Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: make external capabilities explicit and verifiable

kashorinf57ac143c419
diff --git a/packages/caption/README.md b/packages/caption/README.md
index e7cfd0a6..ed69d735 100644
--- a/packages/caption/README.md
+++ b/packages/caption/README.md
@@ -1,6 +1,6 @@
# `@svml/caption`
-Pure composition between Script's untimed Caption Projection and a complete speech time map.
+Pure composition between the common Narrative's untimed Caption Projection and a CompleteSemanticMap.
The authored display region receives the evidence-backed envelope of the speech tokens it owns.
Exact internal correspondences may reuse evidence timing; unresolved display words remain untimed
until a track explicitly chooses a local presentation policy.
diff --git a/packages/caption/package.json b/packages/caption/package.json
index f70c6f28..62e7aa5b 100644
--- a/packages/caption/package.json
+++ b/packages/caption/package.json
@@ -7,7 +7,11 @@
".": "./src/index.ts"
},
"dependencies": {
+ "@svml/contracts": "workspace:*",
"@svml/core": "workspace:*",
+ "@svml/protocol": "workspace:*"
+ },
+ "devDependencies": {
"@svml/script": "workspace:*",
"@svml/speech-align": "workspace:*"
}
diff --git a/packages/caption/src/index.ts b/packages/caption/src/index.ts
index 5fcdea10..039ccd7b 100644
--- a/packages/caption/src/index.ts
+++ b/packages/caption/src/index.ts
@@ -1,4 +1,10 @@
export { CaptionProjectionError } from "./error.js";
+export {
+ captionImplementationDigest,
+ captionManifest,
+ captionModuleRef,
+ captionProducers,
+} from "./manifest.js";
export { planCaptionPresentation } from "./presentation.js";
export { temporalizeCaption } from "./temporalize.js";
export type * from "./types.js";
diff --git a/packages/caption/src/manifest.ts b/packages/caption/src/manifest.ts
new file mode 100644
index 00000000..e81b59b3
--- /dev/null
+++ b/packages/caption/src/manifest.ts
@@ -0,0 +1,36 @@
+import {
+ contractTypes,
+ contractsManifestDigest,
+ contractsModuleRef,
+} from "@svml/contracts";
+import { digestOf } from "@svml/core";
+import type { ModuleManifest, ProducerRef } from "@svml/protocol";
+
+export const captionModuleRef = { name: "@svml/caption", version: "0.0.0-dev" } as const;
+export const captionProducers = {
+ temporalize: { module: captionModuleRef, name: "temporalize-caption" },
+} satisfies Record<string, ProducerRef>;
+export const captionImplementationDigest = digestOf("@svml/caption/temporalize@1");
+
+export const captionManifest: ModuleManifest = {
+ format: "svml.module@0",
+ name: captionModuleRef.name,
+ version: captionModuleRef.version,
+ dependencies: [{ module: contractsModuleRef, digest: contractsManifestDigest }],
+ types: [],
+ surfaces: [],
+ producers: [{
+ name: captionProducers.temporalize.name,
+ inputs: [
+ { name: "narrative", type: contractTypes.narrative },
+ { name: "map", type: contractTypes.completeSemanticMap },
+ ],
+ outputs: [{ name: "caption", type: contractTypes.timedCaptionProjection }],
+ needs: [],
+ implementation: {
+ kind: "registered",
+ locator: "@svml/caption/temporalize",
+ digest: captionImplementationDigest,
+ },
+ }],
+};
diff --git a/packages/caption/src/presentation.ts b/packages/caption/src/presentation.ts
index c6ac0c76..2ac03d7e 100644
--- a/packages/caption/src/presentation.ts
+++ b/packages/caption/src/presentation.ts
@@ -1,11 +1,12 @@
import { digestOf } from "@svml/core";
+import type { Digest } from "@svml/protocol";
import type {
CaptionPresentationMode,
CaptionPresentationPlan,
CaptionPresentationUnit,
TimedCaptionRegion,
-} from "./types.js";
+} from "@svml/contracts";
const DISPLAY_WORD = /[\p{L}\p{M}\p{N}]+(?:['’.-][\p{L}\p{M}\p{N}]+)*/gu;
@@ -121,7 +122,7 @@ function characterFlow(region: TimedCaptionRegion): CaptionPresentationUnit[] {
}
export function planCaptionPresentation(
- projection: { readonly projectionDigest: string; readonly regions: readonly TimedCaptionRegion[] },
+ projection: { readonly projectionDigest: Digest; readonly regions: readonly TimedCaptionRegion[] },
mode: CaptionPresentationMode = "whole",
): CaptionPresentationPlan {
const units = projection.regions.flatMap((region) =>
diff --git a/packages/caption/src/temporalize.ts b/packages/caption/src/temporalize.ts
index f198c806..fe2e7dee 100644
--- a/packages/caption/src/temporalize.ts
+++ b/packages/caption/src/temporalize.ts
@@ -1,6 +1,5 @@
import { digestOf } from "@svml/core";
-import type { ParsedNarrative } from "@svml/script";
-import type { CompleteSpeechTimeMap, TimingQuality } from "@svml/speech-align";
+import type { CompleteSemanticMap, Narrative, TimingQuality } from "@svml/contracts";
import { CaptionProjectionError } from "./error.js";
import type {
@@ -20,8 +19,8 @@ function composedQuality(value: TimingQuality): TimingQuality {
}
export function temporalizeCaption(
- narrative: ParsedNarrative,
- map: CompleteSpeechTimeMap,
+ narrative: Narrative,
+ map: CompleteSemanticMap,
): TimedCaptionProjection {
if (map.semanticIndexDigest !== narrative.semanticIndex.digest) {
throw new CaptionProjectionError(
@@ -69,7 +68,7 @@ export function temporalizeCaption(
};
});
const payload = {
- contract: "svml.timed-caption-projection@0" as const,
+ contract: "svml.timed-caption-projection@1" as const,
semanticIndexDigest: narrative.semanticIndex.digest,
speechTimeMapDigest: map.mapDigest,
text: narrative.captionProjection.text,
diff --git a/packages/caption/src/types.ts b/packages/caption/src/types.ts
index 9a698181..1c24b64e 100644
--- a/packages/caption/src/types.ts
+++ b/packages/caption/src/types.ts
@@ -1,58 +1,8 @@
-import type { TimingQuality } from "@svml/speech-align";
-
-export type TimedCaptionRefinement = {
- readonly id: string;
- readonly display: string;
- readonly displayStart: number;
- readonly displayEnd: number;
- readonly sourceTokenIds: readonly string[];
- readonly startSec: number;
- readonly endSec: number;
- readonly startQuality: TimingQuality;
- readonly endQuality: TimingQuality;
- readonly relation: "exact";
-};
-
-export type TimedCaptionRegion = {
- readonly id: string;
- readonly display: string;
- readonly segmentId: string;
- readonly kind: "identity" | "alias" | "hidden";
- readonly sourceTokenIds: readonly string[];
- readonly startSec: number;
- readonly endSec: number;
- readonly startQuality: TimingQuality;
- readonly endQuality: TimingQuality;
- readonly refinements: readonly TimedCaptionRefinement[];
-};
-
-export type TimedCaptionProjection = {
- readonly contract: "svml.timed-caption-projection@0";
- readonly semanticIndexDigest: string;
- readonly speechTimeMapDigest: string;
- readonly text: string;
- readonly regions: readonly TimedCaptionRegion[];
- readonly projectionDigest: string;
-};
-
-export type CaptionPresentationMode = "whole" | "proportional-word" | "character-flow";
-
-export type CaptionPresentationUnit = {
- readonly id: string;
- readonly regionId: string;
- readonly display: string;
- readonly displayStart: number;
- readonly displayEnd: number;
- readonly startSec: number;
- readonly endSec: number;
- readonly basis: "region-envelope" | "exact-correspondence" | "presentation-estimate";
- readonly timingQuality: TimingQuality;
-};
-
-export type CaptionPresentationPlan = {
- readonly contract: "svml.caption-presentation-plan@0";
- readonly sourceProjectionDigest: string;
- readonly mode: CaptionPresentationMode;
- readonly units: readonly CaptionPresentationUnit[];
- readonly planDigest: string;
-};
+export type {
+ CaptionPresentationMode,
+ CaptionPresentationPlan,
+ CaptionPresentationUnit,
+ TimedCaptionProjection,
+ TimedCaptionRefinement,
+ TimedCaptionRegion,
+} from "@svml/contracts";
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
index 516e2394..18a41fed 100644
--- a/packages/caption/test/caption.test.ts
+++ b/packages/caption/test/caption.test.ts
@@ -2,18 +2,64 @@ import assert from "node:assert/strict";
import test from "node:test";
import { planCaptionPresentation, temporalizeCaption } from "@svml/caption";
+import {
+ sealAlignedTranscriptEvidence,
+ sealProgramSpace,
+ sealSpeechBasis,
+} from "@svml/contracts";
+import type { AlignedTranscriptSegment, Narrative } from "@svml/contracts";
+import { digestOf } from "@svml/core";
import { parseScript } from "@svml/script";
import { locateSpeechTiming } from "@svml/speech-align";
+function locate(narrative: Narrative, durationSec: number, segments: readonly AlignedTranscriptSegment[]) {
+ const programSpace = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec,
+ frameRate: { numerator: 1_000, denominator: 1 },
+ });
+ const audioDigest = digestOf(`caption:audio:${narrative.semanticIndex.digest}`);
+ const basisSegments = narrative.segments.map((segment, index) => ({
+ segmentId: segment.id,
+ startSec: segments[index]!.startSec,
+ endSec: segments[index]!.endSec,
+ sourceArtifactDigest: digestOf(`caption:clip:${segment.id}`),
+ }));
+ const basis = sealSpeechBasis({
+ contract: "svml.speech-basis@1",
+ programSpace,
+ audio: { digest: audioDigest, size: 1, mediaType: "audio/wav", durationSec },
+ visualTrack: { clips: basisSegments.map((segment) => ({
+ segmentId: segment.segmentId,
+ artifact: {
+ digest: segment.sourceArtifactDigest,
+ size: 1,
+ mediaType: "video/mp4",
+ durationSec: segment.endSec - segment.startSec,
+ },
+ startSec: segment.startSec,
+ endSec: segment.endSec,
+ })) },
+ segments: basisSegments,
+ });
+ const evidence = sealAlignedTranscriptEvidence({
+ contract: "svml.aligned-transcript-evidence@1",
+ basisDigest: basis.basisDigest,
+ audioArtifactDigest: basis.audio.digest,
+ programSpaceDigest: basis.programSpace.digest,
+ rawEvidenceArtifactDigest: digestOf("caption:raw-evidence"),
+ durationSec,
+ segments,
+ });
+ return locateSpeechTiming(narrative, basis, evidence);
+}
+
test("Caption temporalization preserves evidence envelopes and labels local estimates", () => {
const narrative = parseScript(
"caption.svml",
"<line><that was insane | what the fuck> <15% off | fifteen percent off></line>",
);
- const map = locateSpeechTiming(narrative, {
- contract: "svml.aligned-transcript-evidence@0",
- durationSec: 2,
- segments: [{
+ const map = locate(narrative, 2, [{
sourceSegmentId: "line",
startSec: 0,
endSec: 2,
@@ -26,8 +72,7 @@ test("Caption temporalization preserves evidence envelopes and labels local esti
{ text: "off", startSec: 1.48, endSec: 1.65 },
],
chars: [],
- }],
- });
+ }]);
const originalMap = structuredClone(map);
const timed = temporalizeCaption(narrative, map);
@@ -57,10 +102,7 @@ test("Caption temporalization preserves evidence envelopes and labels local esti
test("hidden speech owns time but emits no visible presentation unit", () => {
const narrative = parseScript("hidden.svml", "<line>Hello < | um> world.</line>");
- const map = locateSpeechTiming(narrative, {
- contract: "svml.aligned-transcript-evidence@0",
- durationSec: 1,
- segments: [{
+ const map = locate(narrative, 1, [{
sourceSegmentId: "line",
startSec: 0,
endSec: 1,
@@ -70,8 +112,7 @@ test("hidden speech owns time but emits no visible presentation unit", () => {
{ text: "world", startSec: 0.45, endSec: 0.7 },
],
chars: [],
- }],
- });
+ }]);
const timed = temporalizeCaption(narrative, map);
const hidden = timed.regions.find((region) => region.kind === "hidden");
assert.deepEqual([hidden?.startSec, hidden?.endSec], [0.3, 0.4]);
diff --git a/packages/contracts/README.md b/packages/contracts/README.md
new file mode 100644
index 00000000..8d4e5bdb
--- /dev/null
+++ b/packages/contracts/README.md
@@ -0,0 +1,5 @@
+# @svml/contracts
+
+Provider- and Frontend-neutral narrow-waist contracts for the official speech program. This
+package contains data contracts and static schemas only. It does not parse SVML, execute external
+services, align speech or render tracks.
diff --git a/packages/contracts/package.json b/packages/contracts/package.json
new file mode 100644
index 00000000..76624e61
--- /dev/null
+++ b/packages/contracts/package.json
@@ -0,0 +1,13 @@
+{
+ "name": "@svml/contracts",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "dependencies": {
+ "@svml/core": "workspace:*",
+ "@svml/protocol": "workspace:*"
+ }
+}
diff --git a/packages/contracts/src/caption.ts b/packages/contracts/src/caption.ts
new file mode 100644
index 00000000..88f78013
--- /dev/null
+++ b/packages/contracts/src/caption.ts
@@ -0,0 +1,59 @@
+import type { TimingQuality } from "./speech.js";
+import type { Digest } from "@svml/protocol";
+
+export type TimedCaptionRefinement = {
+ readonly id: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly sourceTokenIds: readonly string[];
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+ readonly relation: "exact";
+};
+
+export type TimedCaptionRegion = {
+ readonly id: string;
+ readonly display: string;
+ readonly segmentId: string;
+ readonly kind: "identity" | "alias" | "hidden";
+ readonly sourceTokenIds: readonly string[];
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+ readonly refinements: readonly TimedCaptionRefinement[];
+};
+
+export type TimedCaptionProjection = {
+ readonly contract: "svml.timed-caption-projection@1";
+ readonly semanticIndexDigest: Digest;
+ readonly speechTimeMapDigest: Digest;
+ readonly text: string;
+ readonly regions: readonly TimedCaptionRegion[];
+ readonly projectionDigest: Digest;
+};
+
+export type CaptionPresentationMode = "whole" | "proportional-word" | "character-flow";
+
+export type CaptionPresentationUnit = {
+ readonly id: string;
+ readonly regionId: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly basis: "region-envelope" | "exact-correspondence" | "presentation-estimate";
+ readonly timingQuality: TimingQuality;
+};
+
+export type CaptionPresentationPlan = {
+ readonly contract: "svml.caption-presentation-plan@0";
+ readonly sourceProjectionDigest: Digest;
+ readonly mode: CaptionPresentationMode;
+ readonly units: readonly CaptionPresentationUnit[];
+ readonly planDigest: Digest;
+};
diff --git a/packages/contracts/src/identity.ts b/packages/contracts/src/identity.ts
new file mode 100644
index 00000000..504a00bc
--- /dev/null
+++ b/packages/contracts/src/identity.ts
@@ -0,0 +1,64 @@
+import { digestOf, isDigest } from "@svml/core";
+import type { Digest } from "@svml/protocol";
+
+import type { AlignedTranscriptEvidence, ProgramSpace, SpeechBasis } from "./speech.js";
+
+export function computeProgramSpaceDigest(value: Omit<ProgramSpace, "digest">): Digest {
+ return digestOf(value);
+}
+
+export function computeSpeechBasisDigest(value: Omit<SpeechBasis, "basisDigest">): Digest {
+ return digestOf(value);
+}
+
+export function computeAlignedTranscriptEvidenceDigest(
+ value: Omit<AlignedTranscriptEvidence, "evidenceDigest">,
+): Digest {
+ return digestOf(value);
+}
+
+export function sealProgramSpace(value: Omit<ProgramSpace, "digest">): ProgramSpace {
+ return { ...value, digest: computeProgramSpaceDigest(value) };
+}
+
+export function sealSpeechBasis(value: Omit<SpeechBasis, "basisDigest">): SpeechBasis {
+ return { ...value, basisDigest: computeSpeechBasisDigest(value) };
+}
+
+export function sealAlignedTranscriptEvidence(
+ value: Omit<AlignedTranscriptEvidence, "evidenceDigest">,
+): AlignedTranscriptEvidence {
+ return { ...value, evidenceDigest: computeAlignedTranscriptEvidenceDigest(value) };
+}
+
+export function assertSpeechBasisIdentity(basis: SpeechBasis): void {
+ if (basis.contract !== "svml.speech-basis@1") throw new Error("Unsupported SpeechBasis contract.");
+ const { digest: _programDigest, ...programContent } = basis.programSpace;
+ if (
+ !isDigest(basis.programSpace.digest)
+ || basis.programSpace.digest !== computeProgramSpaceDigest(programContent)
+ ) {
+ throw new Error("ProgramSpace digest does not match its canonical contents.");
+ }
+ const { basisDigest: _basisDigest, ...basisContent } = basis;
+ if (!isDigest(basis.basisDigest) || basis.basisDigest !== computeSpeechBasisDigest(basisContent)) {
+ throw new Error("SpeechBasis digest does not match its canonical contents.");
+ }
+ const { numerator, denominator } = basis.programSpace.frameRate;
+ if (
+ !Number.isSafeInteger(numerator)
+ || numerator <= 0
+ || !Number.isSafeInteger(denominator)
+ || denominator <= 0
+ ) {
+ throw new Error("ProgramSpace frame rate must be a positive rational number.");
+ }
+ if (
+ !Number.isFinite(basis.programSpace.durationSec)
+ || basis.programSpace.durationSec <= 0
+ || basis.audio.durationSec !== basis.programSpace.durationSec
+ || !isDigest(basis.audio.digest)
+ ) {
+ throw new Error("SpeechBasis audio does not match its ProgramSpace.");
+ }
+}
diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts
new file mode 100644
index 00000000..057e3235
--- /dev/null
+++ b/packages/contracts/src/index.ts
@@ -0,0 +1,24 @@
+export type * from "./caption.js";
+export {
+ assertSpeechBasisIdentity,
+ computeAlignedTranscriptEvidenceDigest,
+ computeProgramSpaceDigest,
+ computeSpeechBasisDigest,
+ sealAlignedTranscriptEvidence,
+ sealProgramSpace,
+ sealSpeechBasis,
+} from "./identity.js";
+export {
+ alignedTranscriptEvidenceFields,
+ alignedTranscriptEvidenceSchema,
+ completeSemanticMapSchema,
+ contractsManifest,
+ contractsManifestDigest,
+ contractsModuleRef,
+ contractTypes,
+ narrativeSchema,
+ speechBasisSchema,
+ timedCaptionProjectionSchema,
+} from "./manifest.js";
+export type * from "./narrative.js";
+export type * from "./speech.js";
diff --git a/packages/contracts/src/manifest.ts b/packages/contracts/src/manifest.ts
new file mode 100644
index 00000000..a8a28d6a
--- /dev/null
+++ b/packages/contracts/src/manifest.ts
@@ -0,0 +1,204 @@
+import { computeModuleDigest } from "@svml/core";
+import type { ModuleManifest, TypeRef, ValueSchema } from "@svml/protocol";
+
+export const contractsModuleRef = { name: "@svml/contracts", version: "0.0.0-dev" } as const;
+
+export const contractTypes = {
+ narrative: { module: contractsModuleRef, name: "Narrative" },
+ speechBasis: { module: contractsModuleRef, name: "SpeechBasis" },
+ alignedTranscriptEvidence: { module: contractsModuleRef, name: "AlignedTranscriptEvidence" },
+ completeSemanticMap: { module: contractsModuleRef, name: "CompleteSemanticMap" },
+ timedCaptionProjection: { module: contractsModuleRef, name: "TimedCaptionProjection" },
+} satisfies Record<string, TypeRef>;
+
+const string = { kind: "string", minLength: 1 } as const;
+const number = { kind: "number", minimum: 0 } as const;
+const integer = { kind: "number", integer: true, minimum: 0 } as const;
+const digest = { kind: "string", minLength: 71, maxLength: 71 } as const;
+const object = (fields: Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>, allowUnknown = false): ValueSchema => ({
+ kind: "object",
+ fields,
+ ...(allowUnknown ? { allowUnknown: true } : {}),
+});
+
+const markerBoundary = object({
+ tokenIndex: { schema: integer },
+ structuralPosition: { schema: integer },
+ segmentId: { schema: string, optional: true },
+});
+const markerEdge = object({
+ affinity: { schema: { kind: "string", enum: ["left", "right"] } },
+ boundary: { schema: markerBoundary },
+});
+
+export const narrativeSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.narrative@0" } },
+ segments: { schema: { kind: "array", minItems: 1, items: object({
+ id: { schema: string }, index: { schema: integer }, startAnchorId: { schema: string },
+ endAnchorId: { schema: string }, tokenStart: { schema: integer }, tokenEndExclusive: { schema: integer },
+ }) } },
+ tokens: { schema: { kind: "array", items: object({
+ id: { schema: string }, index: { schema: integer }, segmentId: { schema: string },
+ segmentTokenIndex: { schema: integer }, startAnchorId: { schema: string }, endAnchorId: { schema: string },
+ text: { schema: string }, normalized: { schema: string },
+ }) } },
+ turns: { schema: { kind: "array", items: object({
+ id: { schema: string }, segmentId: { schema: string }, role: { schema: string, optional: true },
+ tokenStart: { schema: integer }, tokenEndExclusive: { schema: integer },
+ }) } },
+ selections: { schema: { kind: "array", items: object({
+ id: { schema: string }, occurrences: { schema: { kind: "array", minItems: 1, items: object({
+ occurrence: { schema: integer }, open: { schema: markerEdge }, close: { schema: markerEdge },
+ }) } },
+ }) } },
+ moments: { schema: { kind: "array", items: object({
+ id: { schema: string }, occurrences: { schema: { kind: "array", minItems: 1, items: object({
+ occurrence: { schema: integer }, affinity: { schema: { kind: "string", enum: ["left", "right"] } },
+ boundary: { schema: markerBoundary },
+ }) } },
+ }) } },
+ captionProjection: { schema: object({
+ contract: { schema: { kind: "literal", value: "svml.caption-projection@0" } },
+ text: { schema: { kind: "string" } },
+ regions: { schema: { kind: "array", items: object({
+ id: { schema: string }, display: { schema: { kind: "string" } }, segmentId: { schema: string },
+ startToken: { schema: integer }, endTokenExclusive: { schema: integer },
+ kind: { schema: { kind: "string", enum: ["identity", "alias", "hidden"] } },
+ refinements: { schema: { kind: "array", items: object({
+ id: { schema: string }, display: { schema: string }, displayStart: { schema: integer },
+ displayEnd: { schema: integer }, startToken: { schema: integer }, endTokenExclusive: { schema: integer },
+ relation: { schema: { kind: "literal", value: "exact" } },
+ }) } },
+ }) } },
+ }) },
+ semanticIndex: { schema: object({
+ contract: { schema: { kind: "literal", value: "svml.semantic-index@0" } },
+ anchors: { schema: { kind: "array", minItems: 2, items: object({
+ id: { schema: string },
+ kind: { schema: { kind: "string", enum: ["segment-start", "token-start", "token-end", "segment-end"] } },
+ segmentId: { schema: string }, tokenId: { schema: string, optional: true },
+ segmentTokenIndex: { schema: integer, optional: true },
+ }) } },
+ digest: { schema: digest },
+ }) },
+ serializations: { schema: object({
+ dialogue: { schema: { kind: "string" } }, speech: { schema: { kind: "string" } },
+ }) },
+});
+
+const artifact = object({
+ digest: { schema: digest }, size: { schema: integer }, mediaType: { schema: string }, durationSec: { schema: number },
+});
+const programSpace = object({
+ contract: { schema: { kind: "literal", value: "svml.program-space@0" } },
+ digest: { schema: digest }, durationSec: { schema: number },
+ frameRate: { schema: object({ numerator: { schema: integer }, denominator: { schema: integer } }) },
+});
+const basisSegment = object({
+ segmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number },
+ sourceArtifactDigest: { schema: digest },
+});
+
+export const speechBasisSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.speech-basis@1" } },
+ basisDigest: { schema: digest }, programSpace: { schema: programSpace }, audio: { schema: artifact },
+ visualTrack: { schema: object({ clips: { schema: { kind: "array", items: object({
+ segmentId: { schema: string }, artifact: { schema: artifact }, startSec: { schema: number }, endSec: { schema: number },
+ }) } } }) },
+ segments: { schema: { kind: "array", minItems: 1, items: basisSegment } },
+});
+
+const wordEvidence = object({
+ text: { schema: { kind: "string" } }, startSec: { schema: number, optional: true },
+ endSec: { schema: number, optional: true }, score: { schema: { kind: "number", minimum: 0, maximum: 1 }, optional: true },
+});
+const charEvidence = object({
+ char: { schema: { kind: "string" } }, wordIndex: { schema: integer }, startSec: { schema: number, optional: true },
+ endSec: { schema: number, optional: true }, score: { schema: { kind: "number", minimum: 0, maximum: 1 }, optional: true },
+});
+export const alignedTranscriptEvidenceFields = {
+ contract: { schema: { kind: "literal", value: "svml.aligned-transcript-evidence@1" } },
+ basisDigest: { schema: digest }, audioArtifactDigest: { schema: digest }, programSpaceDigest: { schema: digest },
+ rawEvidenceArtifactDigest: { schema: digest }, evidenceDigest: { schema: digest }, durationSec: { schema: number },
+ segments: { schema: { kind: "array", minItems: 1, items: object({
+ sourceSegmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number },
+ words: { schema: { kind: "array", items: wordEvidence } },
+ chars: { schema: { kind: "array", items: charEvidence } },
+ speechActivity: { schema: { kind: "array", items: object({ startSec: { schema: number }, endSec: { schema: number } }) }, optional: true },
+ }) } },
+} as const satisfies Readonly<Record<string, { readonly schema: ValueSchema; readonly optional?: boolean }>>;
+export const alignedTranscriptEvidenceSchema: ValueSchema = object(alignedTranscriptEvidenceFields);
+
+const quality = { kind: "string", enum: ["measured", "derived", "estimated"] } as const;
+const timedSegment = object({
+ segmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number },
+ startFrame: { schema: integer }, endFrame: { schema: integer },
+ startQuality: { schema: quality }, endQuality: { schema: quality },
+});
+const timedToken = object({
+ tokenId: { schema: string }, segmentId: { schema: string }, startSec: { schema: number }, endSec: { schema: number },
+ startFrame: { schema: integer }, endFrame: { schema: integer },
+ startQuality: { schema: quality }, endQuality: { schema: quality },
+});
+const semanticPoint = object({
+ identity: { schema: string }, timeSec: { schema: number }, frame: { schema: integer }, quality: { schema: quality },
+});
+const alignmentGroup = object({
+ sourceSegmentId: { schema: string }, sourceTokenIds: { schema: { kind: "array", items: string } },
+ evidenceWordStart: { schema: integer }, evidenceWordEndExclusive: { schema: integer },
+ relation: {
+ schema: {
+ kind: "string",
+ enum: ["exact", "split", "merge", "replacement", "source-omission", "evidence-insertion"],
+ },
+ },
+ cost: { schema: number },
+});
+
+export const completeSemanticMapSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.complete-semantic-map@1" } },
+ semanticIndexDigest: { schema: digest }, basisDigest: { schema: digest }, audioArtifactDigest: { schema: digest },
+ programSpaceDigest: { schema: digest }, evidenceDigest: { schema: digest }, locatorDigest: { schema: digest },
+ quantizationPolicy: { schema: { kind: "literal", value: "nearest-frame" } }, durationSec: { schema: number },
+ segments: { schema: { kind: "array", items: timedSegment } },
+ tokens: { schema: { kind: "array", items: timedToken } },
+ anchors: { schema: { kind: "array", items: semanticPoint } },
+ groups: { schema: { kind: "array", items: alignmentGroup } }, mapDigest: { schema: digest },
+});
+
+const timedCaptionRefinement = object({
+ id: { schema: string }, display: { schema: string }, displayStart: { schema: integer }, displayEnd: { schema: integer },
+ sourceTokenIds: { schema: { kind: "array", items: string } }, startSec: { schema: number }, endSec: { schema: number },
+ startQuality: { schema: quality }, endQuality: { schema: quality },
+ relation: { schema: { kind: "literal", value: "exact" } },
+});
+const timedCaptionRegion = object({
+ id: { schema: string }, display: { schema: { kind: "string" } }, segmentId: { schema: string },
+ kind: { schema: { kind: "string", enum: ["identity", "alias", "hidden"] } },
+ sourceTokenIds: { schema: { kind: "array", items: string } }, startSec: { schema: number }, endSec: { schema: number },
+ startQuality: { schema: quality }, endQuality: { schema: quality },
+ refinements: { schema: { kind: "array", items: timedCaptionRefinement } },
+});
+export const timedCaptionProjectionSchema: ValueSchema = object({
+ contract: { schema: { kind: "literal", value: "svml.timed-caption-projection@1" } },
+ semanticIndexDigest: { schema: digest }, speechTimeMapDigest: { schema: digest }, text: { schema: { kind: "string" } },
+ regions: { schema: { kind: "array", items: timedCaptionRegion } }, projectionDigest: { schema: digest },
+});
+
+export const contractsManifest: ModuleManifest = {
+ format: "svml.module@0",
+ name: contractsModuleRef.name,
+ version: contractsModuleRef.version,
+ dependencies: [],
+ types: [
+ { name: contractTypes.narrative.name, schema: narrativeSchema },
+ { name: contractTypes.speechBasis.name, schema: speechBasisSchema },
+ { name: contractTypes.alignedTranscriptEvidence.name, schema: alignedTranscriptEvidenceSchema },
+ { name: contractTypes.completeSemanticMap.name, schema: completeSemanticMapSchema },
+ { name: contractTypes.timedCaptionProjection.name, schema: timedCaptionProjectionSchema },
+ ],
+ surfaces: [],
+ producers: [],
+};
+
+export const contractsManifestDigest = computeModuleDigest(contractsManifest);
diff --git a/packages/contracts/src/narrative.ts b/packages/contracts/src/narrative.ts
new file mode 100644
index 00000000..0bae7ea1
--- /dev/null
+++ b/packages/contracts/src/narrative.ts
@@ -0,0 +1,111 @@
+export type Affinity = "left" | "right";
+
+export type MarkerBoundary = {
+ readonly tokenIndex: number;
+ readonly structuralPosition: number;
+ readonly segmentId?: string;
+};
+
+export type NarrativeSegment = {
+ readonly id: string;
+ readonly index: number;
+ readonly startAnchorId: string;
+ readonly endAnchorId: string;
+ readonly tokenStart: number;
+ readonly tokenEndExclusive: number;
+};
+
+export type NarrativeToken = {
+ readonly id: string;
+ readonly index: number;
+ readonly segmentId: string;
+ readonly segmentTokenIndex: number;
+ readonly startAnchorId: string;
+ readonly endAnchorId: string;
+ readonly text: string;
+ readonly normalized: string;
+};
+
+export type NarrativeTurn = {
+ readonly id: string;
+ readonly segmentId: string;
+ readonly role?: string;
+ readonly tokenStart: number;
+ readonly tokenEndExclusive: number;
+};
+
+export type NarrativeSelectionOccurrence = {
+ readonly occurrence: number;
+ readonly open: { readonly affinity: Affinity; readonly boundary: MarkerBoundary };
+ readonly close: { readonly affinity: Affinity; readonly boundary: MarkerBoundary };
+};
+
+export type NarrativeSelection = {
+ readonly id: string;
+ readonly occurrences: readonly NarrativeSelectionOccurrence[];
+};
+
+export type NarrativeMomentOccurrence = {
+ readonly occurrence: number;
+ readonly affinity: Affinity;
+ readonly boundary: MarkerBoundary;
+};
+
+export type NarrativeMoment = {
+ readonly id: string;
+ readonly occurrences: readonly NarrativeMomentOccurrence[];
+};
+
+export type CaptionRefinement = {
+ readonly id: string;
+ readonly display: string;
+ readonly displayStart: number;
+ readonly displayEnd: number;
+ readonly startToken: number;
+ readonly endTokenExclusive: number;
+ readonly relation: "exact";
+};
+
+export type CaptionRegion = {
+ readonly id: string;
+ readonly display: string;
+ readonly segmentId: string;
+ readonly startToken: number;
+ readonly endTokenExclusive: number;
+ readonly kind: "identity" | "alias" | "hidden";
+ readonly refinements: readonly CaptionRefinement[];
+};
+
+export type CaptionProjection = {
+ readonly contract: "svml.caption-projection@0";
+ readonly text: string;
+ readonly regions: readonly CaptionRegion[];
+};
+
+export type SemanticAnchor = {
+ readonly id: string;
+ readonly kind: "segment-start" | "token-start" | "token-end" | "segment-end";
+ readonly segmentId: string;
+ readonly tokenId?: string;
+ readonly segmentTokenIndex?: number;
+};
+
+export type Narrative = {
+ readonly contract: "svml.narrative@0";
+ readonly segments: readonly NarrativeSegment[];
+ readonly tokens: readonly NarrativeToken[];
+ readonly turns: readonly NarrativeTurn[];
+ readonly selections: readonly NarrativeSelection[];
+ readonly moments: readonly NarrativeMoment[];
+ readonly captionProjection: CaptionProjection;
+ readonly semanticIndex: {
+ readonly contract: "svml.semantic-index@0";
+ readonly anchors: readonly SemanticAnchor[];
+ readonly digest: Digest;
+ };
+ readonly serializations: {
+ readonly dialogue: string;
+ readonly speech: string;
+ };
+};
+import type { Digest } from "@svml/protocol";
diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts
new file mode 100644
index 00000000..e9edee98
--- /dev/null
+++ b/packages/contracts/src/speech.ts
@@ -0,0 +1,145 @@
+import type { Digest } from "@svml/protocol";
+
+export type TimingQuality = "measured" | "derived" | "estimated";
+
+export type ProgramSpace = {
+ readonly contract: "svml.program-space@0";
+ readonly digest: Digest;
+ readonly durationSec: number;
+ readonly frameRate: {
+ readonly numerator: number;
+ readonly denominator: number;
+ };
+};
+
+export type MediaArtifactRef = {
+ readonly digest: Digest;
+ readonly size: number;
+ readonly mediaType: string;
+ readonly durationSec: number;
+};
+
+export type SpeechBasisSegment = {
+ readonly segmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly sourceArtifactDigest: Digest;
+};
+
+export type SpeechBasis = {
+ readonly contract: "svml.speech-basis@1";
+ readonly basisDigest: Digest;
+ readonly programSpace: ProgramSpace;
+ readonly audio: MediaArtifactRef;
+ readonly visualTrack: {
+ readonly clips: readonly {
+ readonly segmentId: string;
+ readonly artifact: MediaArtifactRef;
+ readonly startSec: number;
+ readonly endSec: number;
+ }[];
+ };
+ readonly segments: readonly SpeechBasisSegment[];
+};
+
+export type SpeechWordEvidence = {
+ readonly text: string;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+};
+
+export type SpeechCharacterEvidence = {
+ readonly char: string;
+ readonly wordIndex: number;
+ readonly startSec?: number;
+ readonly endSec?: number;
+ readonly score?: number;
+};
+
+export type SpeechActivitySpan = {
+ readonly startSec: number;
+ readonly endSec: number;
+};
+
+export type AlignedTranscriptSegment = {
+ readonly sourceSegmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly words: readonly SpeechWordEvidence[];
+ readonly chars: readonly SpeechCharacterEvidence[];
+ readonly speechActivity?: readonly SpeechActivitySpan[];
+};
+
+export type AlignedTranscriptEvidence = {
+ readonly contract: "svml.aligned-transcript-evidence@1";
+ readonly basisDigest: Digest;
+ readonly audioArtifactDigest: Digest;
+ readonly programSpaceDigest: Digest;
+ readonly rawEvidenceArtifactDigest: Digest;
+ readonly evidenceDigest: Digest;
+ readonly durationSec: number;
+ readonly segments: readonly AlignedTranscriptSegment[];
+};
+
+export type AlignmentRelation =
+ | "exact"
+ | "split"
+ | "merge"
+ | "replacement"
+ | "source-omission"
+ | "evidence-insertion";
+
+export type AlignmentGroup = {
+ readonly sourceSegmentId: string;
+ readonly sourceTokenIds: readonly string[];
+ readonly evidenceWordStart: number;
+ readonly evidenceWordEndExclusive: number;
+ readonly relation: AlignmentRelation;
+ readonly cost: number;
+};
+
+export type TimedSpeechToken = {
+ readonly tokenId: string;
+ readonly segmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startFrame: number;
+ readonly endFrame: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+};
+
+export type TimedSpeechSegment = {
+ readonly segmentId: string;
+ readonly startSec: number;
+ readonly endSec: number;
+ readonly startFrame: number;
+ readonly endFrame: number;
+ readonly startQuality: TimingQuality;
+ readonly endQuality: TimingQuality;
+};
+
+export type SemanticTimePoint = {
+ readonly identity: string;
+ readonly timeSec: number;
+ readonly frame: number;
+ readonly quality: TimingQuality;
+};
+
+export type CompleteSemanticMap = {
+ readonly contract: "svml.complete-semantic-map@1";
+ readonly semanticIndexDigest: Digest;
+ readonly basisDigest: Digest;
+ readonly audioArtifactDigest: Digest;
+ readonly programSpaceDigest: Digest;
+ readonly evidenceDigest: Digest;
+ readonly locatorDigest: Digest;
+ readonly quantizationPolicy: "nearest-frame";
+ readonly durationSec: number;
+ readonly segments: readonly TimedSpeechSegment[];
+ readonly tokens: readonly TimedSpeechToken[];
+ readonly anchors: readonly SemanticTimePoint[];
+ readonly groups: readonly AlignmentGroup[];
+ readonly mapDigest: Digest;
+};
diff --git a/packages/core/README.md b/packages/core/README.md
index c4425547..c092b43c 100644
--- a/packages/core/README.md
+++ b/packages/core/README.md
@@ -6,6 +6,11 @@ The package accepts an already resolved module closure, typed authored modules a
finite build plan. It validates identities, schemas, immutable records, Needs, Receipts and
Derivations, then advances a serializable `BuildState` with pure `reduce(state, event)` calls.
+Derivations bind Producer and implementation identity, input/output Record digests, Need request
+digests and the accepted event digest. Needs carry the conformance floor inherited from their
+inputs, so an exact external response cannot wash a substitute upstream back to exact. An exact
+Need above a substitute floor fails before an external Handler can run.
+
It generically verifies Surface declarations in the same immutable module closure, but never
parses or dispatches one. It does not parse source, resolve package locations, execute
implementations, access artifact bytes, select providers or search for a workflow. Those
diff --git a/packages/core/src/provenance.ts b/packages/core/src/provenance.ts
new file mode 100644
index 00000000..56ac1b6e
--- /dev/null
+++ b/packages/core/src/provenance.ts
@@ -0,0 +1,38 @@
+import type {
+ Derivation,
+ DerivationId,
+ Digest,
+ Need,
+ Receipt,
+ ReceiptId,
+} from "@svml/protocol";
+
+import { digestOf } from "./canonical.js";
+
+export function commandId(build: string, kind: string, subject: string): string {
+ return `command:${digestOf({ build, kind, subject })}`;
+}
+
+export function eventDigest(event: unknown): Digest {
+ return digestOf(event);
+}
+
+export function needRequestDigest(
+ need: Pick<Need, "wants" | "constraints" | "result" | "accepts" | "conformanceFloor">,
+): Digest {
+ return digestOf({
+ wants: need.wants,
+ constraints: need.constraints,
+ result: need.result,
+ accepts: need.accepts,
+ conformanceFloor: need.conformanceFloor,
+ });
+}
+
+export function derivationId(derivation: Omit<Derivation, "id">): DerivationId {
+ return `derivation:${digestOf(derivation)}`;
+}
+
+export function receiptId(receipt: Omit<Receipt, "id">): ReceiptId {
+ return `receipt:${digestOf(receipt)}`;
+}
diff --git a/packages/core/src/reducer.ts b/packages/core/src/reducer.ts
index 1f80369d..136309fb 100644
--- a/packages/core/src/reducer.ts
+++ b/packages/core/src/reducer.ts
@@ -22,18 +22,17 @@ import { canonicalize, digestOf, recordDigest } from "./canonical.js";
import { CoreError, invariant } from "./error.js";
import { resolveProducer, resolveType, sealRecord } from "./link.js";
import { producerStep, validatePlan } from "./plan.js";
+import {
+ commandId,
+ derivationId,
+ eventDigest,
+ needRequestDigest,
+ receiptId,
+} from "./provenance.js";
import { producerKey } from "./reference.js";
import { validateStoredValue } from "./schema.js";
import { verifyBuildState } from "./verify.js";
-function commandId(build: string, kind: string, subject: string): string {
- return `command:${digestOf({ build, kind, subject })}`;
-}
-
-function eventDigest(event: BuildEvent): ReturnType<typeof digestOf> {
- return digestOf(event);
-}
-
function withoutCommand(state: BuildState, id: string): readonly CoreCommand[] {
return state.outstanding.filter((command) => command.id !== id);
}
@@ -46,6 +45,10 @@ function inheritedConformance(records: readonly TypedRecord[]): Conformance {
return records.some((record) => record.conformance === "substitute") ? "substitute" : "exact";
}
+function effectiveConformance(left: Conformance, right: Conformance): Conformance {
+ return left === "substitute" || right === "substitute" ? "substitute" : "exact";
+}
+
function inputRecords(state: BuildState, command: InvokeProducerCommand): TypedRecord[] {
return Object.values(command.inputs).map((id) => {
const record = state.records.find((item) => item.id === id);
@@ -91,62 +94,55 @@ function acceptProducerEvent(
const inputs = inputRecords(state, command);
const conformance = inheritedConformance(inputs);
- const derivationId = `derivation:${digestOf({
- build: state.id,
- command: command.id,
- event: eventDigest(event),
- inputs: inputs.map((record) => ({ id: record.id, digest: record.digest })),
- })}`;
-
- const outputs: TypedRecord[] = producer.outputs.map((port) => {
+ const outputDrafts = producer.outputs.map((port) => {
const id = step.outputs[port.name];
const rawValue = event.outputs[port.name];
invariant(id !== undefined, "MISSING_OUTPUT_BINDING", `${step.id}.${port.name} is not bound`);
invariant(rawValue !== undefined, "MISSING_OUTPUT_VALUE", `${step.id}.${port.name} returned no value`);
const value = normalizeStoredValue(rawValue);
validateStoredValue(value, resolveType(state.program.closure, port.type).schema, `$output.${step.id}.${port.name}`);
- return sealRecord({
- id,
- type: port.type,
- value,
- conformance,
- origin: { kind: "derived", derivation: derivationId },
- });
+ return { id, type: port.type, value, digest: recordDigest(port.type, value) };
});
- const needs: Need[] = producer.needs.map((port) => {
+ const needDrafts = producer.needs.map((port) => {
const binding = step.needs[port.name];
const constraints = event.needs[port.name];
invariant(binding !== undefined, "MISSING_NEED_BINDING", `${step.id}.${port.name} is not bound`);
invariant(constraints !== undefined, "MISSING_NEED_VALUE", `${step.id}.${port.name} returned no constraints`);
const normalized = canonicalize(constraints);
- const requestDigest = digestOf({
+ const request = {
wants: port.wants,
constraints: normalized,
- requestedBy: derivationId,
result: binding.result,
accepts: binding.accepts,
- });
+ conformanceFloor: conformance,
+ } as const;
return {
id: binding.id,
- wants: port.wants,
- constraints: normalized,
- requestedBy: derivationId,
- result: binding.result,
- accepts: binding.accepts,
- requestDigest,
+ ...request,
+ requestDigest: needRequestDigest(request),
};
});
- const derivation: Derivation = {
- id: derivationId,
+ const derivationDraft: Omit<Derivation, "id"> = {
step: step.id,
producer: step.producer,
implementationDigest: producer.implementation.digest,
- inputs: inputs.map((record) => record.id),
- outputs: outputs.map((record) => record.id),
- needs: needs.map((need) => need.id),
+ inputs: inputs.map((record) => ({ id: record.id, digest: record.digest })),
+ outputs: outputDrafts.map(({ id, digest }) => ({ id, digest })),
+ needs: needDrafts.map((need) => ({ id: need.id, requestDigest: need.requestDigest })),
+ event: { id: event.id, digest: eventDigest(event) },
};
+ const id = derivationId(derivationDraft);
+ const derivation: Derivation = { id, ...derivationDraft };
+ const outputs: TypedRecord[] = outputDrafts.map((output) => sealRecord({
+ id: output.id,
+ type: output.type,
+ value: output.value,
+ conformance,
+ origin: { kind: "derived", derivation: id },
+ }));
+ const needs: Need[] = needDrafts.map((need) => ({ ...need, requestedBy: id }));
return {
...state,
@@ -154,7 +150,7 @@ function acceptProducerEvent(
needs: [...state.needs, ...needs],
derivations: [...state.derivations, derivation],
steps: state.steps.map((item) =>
- item.id === step.id ? { id: item.id, status: "complete", derivation: derivationId } : item,
+ item.id === step.id ? { id: item.id, status: "complete", derivation: id } : item,
),
outstanding: withoutCommand(state, command.id),
acceptedEvents: addAcceptedEvent(state, event),
@@ -181,7 +177,8 @@ function acceptNeedEvent(
need.id,
);
invariant(
- need.accepts === "substitute" || event.conformance === "exact",
+ need.accepts === "substitute"
+ || effectiveConformance(need.conformanceFloor, event.conformance) === "exact",
"SUBSTITUTE_NOT_ACCEPTED",
`${need.id} requires exact fulfillment`,
need.id,
@@ -191,33 +188,26 @@ function acceptNeedEvent(
validateStoredValue(value, resolveType(state.program.closure, need.wants).schema, `$need.${need.id}`);
const outputDigest = recordDigest(need.wants, value);
const metadata = canonicalize(event.metadata);
- const receiptId = `receipt:${digestOf({
+ const conformance = effectiveConformance(need.conformanceFloor, event.conformance);
+ const receiptDraft: Omit<Receipt, "id"> = {
need: need.id,
requestDigest: event.requestDigest,
fulfiller: event.fulfiller,
- conformance: event.conformance,
- delivery: event.delivery,
- output: need.result,
- outputDigest,
- metadata,
- })}`;
- const receipt: Receipt = {
- id: receiptId,
- need: need.id,
- requestDigest: need.requestDigest,
- fulfiller: event.fulfiller,
- conformance: event.conformance,
+ fulfillmentConformance: event.conformance,
+ conformance,
delivery: event.delivery,
output: need.result,
outputDigest,
metadata,
+ event: { id: event.id, digest: eventDigest(event) },
};
+ const receipt: Receipt = { id: receiptId(receiptDraft), ...receiptDraft };
const record: TypedRecord = {
id: need.result,
type: need.wants,
value,
digest: outputDigest,
- conformance: event.conformance,
+ conformance,
origin: { kind: "observed", receipt: receipt.id },
};
@@ -283,6 +273,28 @@ function goalsComplete(state: BuildState): boolean {
function schedule(state: BuildState): CoreTransition {
if (state.status !== "active") return { state, commands: [] };
+ const impossibleNeed = state.needs.find((need) =>
+ need.accepts === "exact"
+ && need.conformanceFloor === "substitute"
+ && !state.records.some((record) => record.id === need.result));
+ if (impossibleNeed !== undefined) {
+ return {
+ state: {
+ ...state,
+ status: "failed",
+ outstanding: [],
+ diagnostics: [
+ ...state.diagnostics,
+ {
+ code: "CONFORMANCE_FLOOR_UNSATISFIABLE",
+ message: `${impossibleNeed.id} requires exact output from substitute inputs`,
+ subject: impossibleNeed.id,
+ },
+ ],
+ },
+ commands: [],
+ };
+ }
if (state.outstanding.length > 0) return { state, commands: state.outstanding };
if (goalsComplete(state)) {
diff --git a/packages/core/src/verify.ts b/packages/core/src/verify.ts
index a14e14c1..4b7c1c05 100644
--- a/packages/core/src/verify.ts
+++ b/packages/core/src/verify.ts
@@ -11,6 +11,7 @@ import { digestOf, isDigest } from "./canonical.js";
import { invariant } from "./error.js";
import { link, resolveProducer, verifyRecord } from "./link.js";
import { producerStep, validatePlan } from "./plan.js";
+import { commandId, derivationId, needRequestDigest, receiptId } from "./provenance.js";
import { sameType } from "./reference.js";
function unique<T>(items: readonly T[], key: (item: T) => string, kind: string): void {
@@ -22,20 +23,14 @@ function unique<T>(items: readonly T[], key: (item: T) => string, kind: string):
}
}
-function needDigest(need: Need): ReturnType<typeof digestOf> {
- return digestOf({
- wants: need.wants,
- constraints: need.constraints,
- requestedBy: need.requestedBy,
- result: need.result,
- accepts: need.accepts,
- });
-}
-
function findRecord(state: BuildState, id: string): TypedRecord | undefined {
return state.records.find((record) => record.id === id);
}
+function validConformance(value: string): boolean {
+ return value === "exact" || value === "substitute";
+}
+
function verifyDerivation(state: BuildState, derivation: Derivation): void {
const step = producerStep(state.plan, derivation.step);
const producer = resolveProducer(state.program.closure, derivation.producer);
@@ -54,36 +49,82 @@ function verifyDerivation(state: BuildState, derivation: Derivation): void {
derivation.id,
);
invariant(
- JSON.stringify([...derivation.inputs].sort()) ===
+ JSON.stringify(derivation.inputs.map((item) => item.id).sort()) ===
JSON.stringify(Object.values(step.inputs).sort()),
"DERIVATION_INPUT_MISMATCH",
`${derivation.id} input records do not match its step`,
derivation.id,
);
invariant(
- JSON.stringify([...derivation.outputs].sort()) ===
+ JSON.stringify(derivation.outputs.map((item) => item.id).sort()) ===
JSON.stringify(Object.values(step.outputs).sort()),
"DERIVATION_OUTPUT_MISMATCH",
`${derivation.id} output records do not match its step`,
derivation.id,
);
invariant(
- JSON.stringify([...derivation.needs].sort()) ===
+ JSON.stringify(derivation.needs.map((item) => item.id).sort()) ===
JSON.stringify(Object.values(step.needs).map((binding) => binding.id).sort()),
"DERIVATION_NEED_MISMATCH",
`${derivation.id} needs do not match its step`,
derivation.id,
);
+ for (const binding of [...derivation.inputs, ...derivation.outputs]) {
+ const record = findRecord(state, binding.id);
+ invariant(record !== undefined, "DERIVATION_RECORD_MISSING", `${binding.id} is missing`, derivation.id);
+ invariant(record.digest === binding.digest, "DERIVATION_RECORD_DIGEST", `${binding.id} digest differs`, derivation.id);
+ }
+ for (const binding of derivation.needs) {
+ const need = state.needs.find((item) => item.id === binding.id);
+ invariant(need !== undefined, "DERIVATION_NEED_MISSING", `${binding.id} is missing`, derivation.id);
+ invariant(
+ need.requestDigest === binding.requestDigest,
+ "DERIVATION_NEED_DIGEST",
+ `${binding.id} request digest differs`,
+ derivation.id,
+ );
+ }
+ const accepted = state.acceptedEvents.find((item) => item.id === derivation.event.id);
+ invariant(
+ accepted?.digest === derivation.event.digest,
+ "DERIVATION_EVENT_MISMATCH",
+ `${derivation.id} event differs`,
+ derivation.id,
+ );
+ const { id: _id, ...content } = derivation;
+ invariant(
+ derivation.id === derivationId(content),
+ "DERIVATION_DIGEST_MISMATCH",
+ `${derivation.id} content does not match its identity`,
+ derivation.id,
+ );
}
function verifyReceipt(state: BuildState, receipt: Receipt): void {
const need = state.needs.find((item) => item.id === receipt.need);
invariant(need !== undefined, "RECEIPT_UNKNOWN_NEED", `${receipt.id} references an unknown need`);
+ invariant(receipt.fulfiller.length > 0, "EMPTY_FULFILLER", `${receipt.id} fulfiller is empty`);
+ invariant(validConformance(receipt.fulfillmentConformance), "INVALID_CONFORMANCE", receipt.id);
+ invariant(validConformance(receipt.conformance), "INVALID_CONFORMANCE", receipt.id);
+ invariant(
+ ["executed", "cache", "manual", "provided"].includes(receipt.delivery),
+ "INVALID_DELIVERY",
+ receipt.id,
+ );
invariant(
receipt.requestDigest === need.requestDigest,
"REQUEST_DIGEST_MISMATCH",
`${receipt.id} request digest does not match`,
);
+ invariant(
+ receipt.conformance === (
+ need.conformanceFloor === "substitute" || receipt.fulfillmentConformance === "substitute"
+ ? "substitute"
+ : "exact"
+ ),
+ "CONFORMANCE_FLOOR_MISMATCH",
+ `${receipt.id} does not inherit its Need conformance floor`,
+ );
invariant(
need.accepts === "substitute" || receipt.conformance === "exact",
"SUBSTITUTE_NOT_ACCEPTED",
@@ -91,6 +132,9 @@ function verifyReceipt(state: BuildState, receipt: Receipt): void {
);
const record = findRecord(state, receipt.output);
invariant(record !== undefined, "RECEIPT_OUTPUT_MISSING", `${receipt.id} output is missing`);
+ invariant(receipt.output === need.result, "RECEIPT_OUTPUT_BINDING", `${receipt.id} output is not its Need result`);
+ invariant(sameType(record.type, need.wants), "RECEIPT_OUTPUT_TYPE", `${receipt.id} output type differs`);
+ invariant(record.conformance === receipt.conformance, "RECEIPT_OUTPUT_CONFORMANCE", receipt.id);
invariant(record.digest === receipt.outputDigest, "RECEIPT_OUTPUT_MISMATCH", `${receipt.id} output differs`);
invariant(
record.origin.kind === "observed" && record.origin.receipt === receipt.id,
@@ -98,17 +142,13 @@ function verifyReceipt(state: BuildState, receipt: Receipt): void {
`${receipt.id} output has another origin`,
);
invariant(
- receipt.id ===
- `receipt:${digestOf({
- need: receipt.need,
- requestDigest: receipt.requestDigest,
- fulfiller: receipt.fulfiller,
- conformance: receipt.conformance,
- delivery: receipt.delivery,
- output: receipt.output,
- outputDigest: receipt.outputDigest,
- metadata: receipt.metadata,
- })}`,
+ state.acceptedEvents.some((item) => item.id === receipt.event.id && item.digest === receipt.event.digest),
+ "RECEIPT_EVENT_MISMATCH",
+ `${receipt.id} accepted event differs`,
+ );
+ const { id: _id, ...content } = receipt;
+ invariant(
+ receipt.id === receiptId(content),
"RECEIPT_DIGEST_MISMATCH",
`${receipt.id} content does not match its identity`,
);
@@ -122,16 +162,34 @@ function verifyOutstanding(state: BuildState, command: CoreCommand): void {
const step = state.steps.find((item) => item.id === command.step);
invariant(step?.status === "pending", "OUTSTANDING_STEP_COMPLETE", `${command.step} is complete`);
const planned = producerStep(state.plan, command.step);
- invariant(planned.producer.name === command.producer.name, "COMMAND_PRODUCER_MISMATCH", command.id);
+ const expected: CoreCommand = {
+ kind: "invoke-producer",
+ id: commandId(state.id, "producer", planned.id),
+ step: planned.id,
+ producer: planned.producer,
+ inputs: planned.inputs,
+ };
+ invariant(digestOf(command) === digestOf(expected), "COMMAND_PRODUCER_MISMATCH", command.id);
return;
}
const need = state.needs.find((item) => item.id === command.need.id);
invariant(need !== undefined, "OUTSTANDING_NEED_UNKNOWN", `${command.need.id} is unknown`);
invariant(findRecord(state, need.result) === undefined, "OUTSTANDING_NEED_COMPLETE", `${need.id} is fulfilled`);
+ const expected: CoreCommand = {
+ kind: "fulfill-need",
+ id: commandId(state.id, "need", need.id),
+ need,
+ };
+ invariant(digestOf(command) === digestOf(expected), "COMMAND_NEED_MISMATCH", command.id);
}
export function verifyBuildState(state: BuildState): void {
invariant(state.format === "svml.build@0", "UNSUPPORTED_BUILD", "unsupported build state format");
+ invariant(
+ state.status === "active" || state.status === "complete" || state.status === "failed",
+ "INVALID_BUILD_STATUS",
+ "build status is invalid",
+ );
invariant(isDigest(state.id), "INVALID_DIGEST", "build id is invalid");
const linked = link(state.program.closure, state.program.modules);
@@ -181,7 +239,10 @@ export function verifyBuildState(state: BuildState): void {
"build step state does not match the plan",
);
- for (const record of state.records) verifyRecord(state.program.closure, record);
+ for (const record of state.records) {
+ invariant(validConformance(record.conformance), "INVALID_CONFORMANCE", record.id);
+ verifyRecord(state.program.closure, record);
+ }
for (const authored of state.program.records) {
const record = findRecord(state, authored.id);
invariant(record?.digest === authored.digest, "AUTHORED_RECORD_CHANGED", `${authored.id} changed`);
@@ -203,11 +264,11 @@ export function verifyBuildState(state: BuildState): void {
`${record.id} has unknown derivation ${origin.derivation}`,
);
invariant(
- derivation.outputs.includes(record.id),
+ derivation.outputs.some((item) => item.id === record.id && item.digest === record.digest),
"DERIVED_ORIGIN_MISMATCH",
`${record.id} is not an output of ${origin.derivation}`,
);
- const inputs = derivation.inputs.map((id) => findRecord(state, id));
+ const inputs = derivation.inputs.map((binding) => findRecord(state, binding.id));
invariant(inputs.every((item) => item !== undefined), "DERIVATION_INPUT_MISSING", derivation.id);
const expected = inputs.some((item) => item?.conformance === "substitute") ? "substitute" : "exact";
invariant(record.conformance === expected, "CONFORMANCE_NOT_PROPAGATED", `${record.id} conformance differs`);
@@ -219,9 +280,27 @@ export function verifyBuildState(state: BuildState): void {
}
for (const need of state.needs) {
- invariant(need.requestDigest === needDigest(need), "NEED_DIGEST_MISMATCH", `${need.id} digest differs`);
+ invariant(need.accepts === "exact" || need.accepts === "substitute", "INVALID_NEED_ACCEPTANCE", need.id);
+ invariant(validConformance(need.conformanceFloor), "INVALID_CONFORMANCE", need.id);
+ invariant(
+ need.requestDigest === needRequestDigest(need),
+ "NEED_DIGEST_MISMATCH",
+ `${need.id} digest differs`,
+ );
const derivation = state.derivations.find((item) => item.id === need.requestedBy);
- invariant(derivation?.needs.includes(need.id), "NEED_ORIGIN_MISMATCH", `${need.id} has no derivation`);
+ invariant(
+ derivation?.needs.some((item) => item.id === need.id && item.requestDigest === need.requestDigest),
+ "NEED_ORIGIN_MISMATCH",
+ `${need.id} has no derivation`,
+ );
+ const inherited = derivation?.inputs.some(
+ (binding) => findRecord(state, binding.id)?.conformance === "substitute",
+ ) ? "substitute" : "exact";
+ invariant(
+ need.conformanceFloor === inherited,
+ "CONFORMANCE_FLOOR_MISMATCH",
+ `${need.id} conformance floor differs from its Producer inputs`,
+ );
}
for (const receipt of state.receipts) verifyReceipt(state, receipt);
for (const derivation of state.derivations) verifyDerivation(state, derivation);
diff --git a/packages/driver-node/README.md b/packages/driver-node/README.md
index e3d75615..6c428acf 100644
--- a/packages/driver-node/README.md
+++ b/packages/driver-node/README.md
@@ -3,7 +3,7 @@
Node runtime for executing Core commands without adding workflow semantics to Core.
- `HostRegistry` registers deterministic Producer implementations locked by implementation digest.
-- `ProviderRegistry` registers external capabilities that may use APIs, credentials, queues or local runtimes.
+- `ProviderRegistry` binds an already explicit external capability to endpoints that may use APIs, credentials, queues or local runtimes.
- A Producer emits a typed `Need`; it never reads provider configuration directly.
- One matching Provider may fulfill a Need. Multiple matches require an explicit runtime `bind()`.
- Provider identity becomes the Receipt fulfiller; handlers return only value, conformance, delivery and metadata.
@@ -11,6 +11,11 @@ Node runtime for executing Core commands without adding workflow semantics to Co
Build state is serializable. Missing providers, generation latency and transient provider errors pause a
build without replaying completed Producers.
+Outstanding Commands are not trusted persistence. Serialization drops them and Core regenerates
+the exact commands from verified facts on resume, before any Handler can run. The Registry never
+turns a generic speaker request into a model choice: Seedance, WhisperX or another external method
+must already be present as a specific Need in the locked BuildPlan.
+
The Driver still does not load implementation code from a package locator. A future sandboxed executor
must be designed and audited before third-party implementations can run. Frontend and Import Prologue
behavior remain outside this package.
diff --git a/packages/driver-node/package.json b/packages/driver-node/package.json
index a7fcd29f..134226cc 100644
--- a/packages/driver-node/package.json
+++ b/packages/driver-node/package.json
@@ -9,5 +9,12 @@
"dependencies": {
"@svml/core": "workspace:*",
"@svml/protocol": "workspace:*"
+ },
+ "devDependencies": {
+ "@svml/caption": "workspace:*",
+ "@svml/contracts": "workspace:*",
+ "@svml/script": "workspace:*",
+ "@svml/speech-align": "workspace:*",
+ "@svml/whisperx": "workspace:*"
}
}
diff --git a/packages/driver-node/src/persistence.ts b/packages/driver-node/src/persistence.ts
index e4c2533f..70573668 100644
--- a/packages/driver-node/src/persistence.ts
+++ b/packages/driver-node/src/persistence.ts
@@ -3,11 +3,14 @@ import type { BuildState } from "@svml/protocol";
export function serializeBuildState(state: BuildState): string {
verifyBuildState(state);
- return canonicalStringify(state);
+ // Outstanding commands are derived scheduling output, not trusted durable state.
+ // A resumed Core deterministically regenerates them from completed facts.
+ return canonicalStringify({ ...state, outstanding: [] });
}
export function parseBuildState(text: string): BuildState {
- const value = JSON.parse(text) as BuildState;
+ const parsed = JSON.parse(text) as BuildState;
+ const value = { ...parsed, outstanding: [] };
verifyBuildState(value);
return value;
}
diff --git a/packages/driver-node/test/driver.test.ts b/packages/driver-node/test/driver.test.ts
index 4c3f7888..4b705540 100644
--- a/packages/driver-node/test/driver.test.ts
+++ b/packages/driver-node/test/driver.test.ts
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
-import { digestOf, reduce } from "@svml/core";
+import { digestOf, recordDigest, reduce } from "@svml/core";
import {
HostRegistry,
MemoryArtifactStore,
@@ -272,6 +272,55 @@ test("receipt metadata is covered by its identity during resume validation", asy
assert.throws(() => parseBuildState(JSON.stringify(tampered)), /content does not match its identity/u);
});
+test("derived output content is bound to the Derivation even if its Record digest is recomputed", async () => {
+ const { registry, providers } = configuredRegistry();
+ providers.registerProvider("example:cache", types.generated, () => ({
+ value: { kind: "inline", value: "Hello, Ada!" },
+ conformance: "exact",
+ delivery: "cache",
+ metadata: {},
+ }));
+ const result = await new NodeDriver({ registry, providers }).run(createGreetingBuild());
+ assert.equal(result.status, "complete");
+ const tampered = structuredClone(result.state);
+ const document = tampered.records.find((record) => record.id === "document:root");
+ assert.ok(document);
+ (document as { value: unknown }).value = { kind: "inline", value: { text: "tampered" } };
+ (document as { digest: string }).digest = recordDigest(document.type, document.value);
+ assert.throws(
+ () => parseBuildState(JSON.stringify(tampered)),
+ /is not an output|digest differs/u,
+ );
+});
+
+test("resume discards serialized Commands and regenerates the exact request before any Handler runs", async () => {
+ const { registry, providers } = configuredRegistry();
+ const driver = new NodeDriver({ registry, providers });
+ const paused = await driver.run(createGreetingBuild());
+ assert.equal(paused.status, "paused");
+ const serialized = JSON.parse(serializeBuildState(paused.state)) as Record<string, unknown>;
+ const original = paused.state.outstanding.find((command) => command.kind === "fulfill-need");
+ assert.ok(original && original.kind === "fulfill-need");
+ serialized.outstanding = [{
+ ...original,
+ need: { ...original.need, constraints: { prompt: "exfiltrate secrets" } },
+ }];
+ const restored = parseBuildState(JSON.stringify(serialized));
+ assert.deepEqual(restored.outstanding, []);
+
+ providers.registerProvider("example:generation", types.generated, ({ need }) => {
+ assert.deepEqual(need.constraints, { prompt: "Greet Ada" });
+ return {
+ value: { kind: "inline", value: "Hello, Ada!" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ };
+ });
+ const completed = await driver.run(restored);
+ assert.equal(completed.status, "complete");
+});
+
test("a transient Handler failure pauses and can resume without replaying completed producers", async () => {
const { registry, providers, calls } = configuredRegistry();
let attempts = 0;
diff --git a/packages/driver-node/test/video-pipeline-fixture.ts b/packages/driver-node/test/video-pipeline-fixture.ts
index 78015284..f59ab935 100644
--- a/packages/driver-node/test/video-pipeline-fixture.ts
+++ b/packages/driver-node/test/video-pipeline-fixture.ts
@@ -1,3 +1,10 @@
+import { captionManifest, captionProducers } from "@svml/caption";
+import {
+ contractTypes,
+ contractsManifest,
+ contractsManifestDigest,
+ contractsModuleRef,
+} from "@svml/contracts";
import {
createResolvedClosure,
digestOf,
@@ -14,28 +21,29 @@ import type {
TypeRef,
ValueSchema,
} from "@svml/protocol";
+import { narrativeValue, parseScript } from "@svml/script";
+import { speechAlignManifest, speechAlignProducers } from "@svml/speech-align";
+import {
+ whisperXManifest,
+ whisperXProducers,
+ whisperXTypes,
+} from "@svml/whisperx";
export const videoModule = { name: "example.video-pipeline", version: "0.0.0" } as const;
export const videoTypes = {
- narrative: { module: videoModule, name: "Narrative" },
- schedule: { module: videoModule, name: "SpeechSchedule" },
- basis: { module: videoModule, name: "SpeechBasis" },
- evidence: { module: videoModule, name: "AlignedTranscriptEvidence" },
- speechMap: { module: videoModule, name: "CompleteSpeechTimeMap" },
- caption: { module: videoModule, name: "TimedCaption" },
+ estimate: { module: videoModule, name: "OfficialSpeechDurationEstimate" },
+ seedanceMiniMedia: { module: videoModule, name: "SeedanceMiniSpeechMedia" },
} satisfies Record<string, TypeRef>;
export const videoProducers = {
- requestEstimate: { module: videoModule, name: "request-estimate" },
- requestGeneration: { module: videoModule, name: "request-generation" },
- requestRecognition: { module: videoModule, name: "request-recognition" },
- locateSpeech: { module: videoModule, name: "locate-speech" },
- temporalizeCaption: { module: videoModule, name: "temporalize-caption" },
+ requestEstimate: { module: videoModule, name: "request-official-speech-estimate" },
+ requestSeedanceMini: { module: videoModule, name: "request-seedance-mini-speech" },
+ assembleBasis: { module: videoModule, name: "assemble-speech-basis" },
} satisfies Record<string, ProducerRef>;
export const videoImplementations = Object.fromEntries(
- Object.entries(videoProducers).map(([name]) => [name, digestOf(`example.video-pipeline/${name}@0`)]),
+ Object.entries(videoProducers).map(([name]) => [name, digestOf(`example.video-pipeline/${name}@1`)]),
) as Readonly<Record<keyof typeof videoProducers, ReturnType<typeof digestOf>>>;
const string = { kind: "string", minLength: 1 } as const;
@@ -49,25 +57,26 @@ export const videoManifest: ModuleManifest = {
format: "svml.module@0",
name: videoModule.name,
version: videoModule.version,
- dependencies: [],
+ dependencies: [{ module: contractsModuleRef, digest: contractsManifestDigest }],
types: [
- { name: videoTypes.narrative.name, schema: object({ speech: { schema: string }, display: { schema: string } }) },
- { name: videoTypes.schedule.name, schema: object({ durationSec: { schema: number } }) },
- { name: videoTypes.basis.name, schema: object({ audio: { schema: string }, visual: { schema: string } }) },
- { name: videoTypes.evidence.name, schema: object({ transcript: { schema: string } }) },
- { name: videoTypes.speechMap.name, schema: object({ startSec: { schema: number }, endSec: { schema: number } }) },
+ { name: videoTypes.estimate.name, schema: object({ durationSec: { schema: number } }) },
{
- name: videoTypes.caption.name,
- schema: object({ text: { schema: string }, startSec: { schema: number }, endSec: { schema: number } }),
+ name: videoTypes.seedanceMiniMedia.name,
+ schema: object({
+ model: { schema: { kind: "literal", value: "mini" } },
+ audioDigest: { schema: string },
+ visualDigest: { schema: string },
+ durationSec: { schema: number },
+ }),
},
],
surfaces: [],
producers: [
{
name: videoProducers.requestEstimate.name,
- inputs: [{ name: "narrative", type: videoTypes.narrative }],
+ inputs: [{ name: "narrative", type: contractTypes.narrative }],
outputs: [],
- needs: [{ name: "schedule", wants: videoTypes.schedule }],
+ needs: [{ name: "estimate", wants: videoTypes.estimate }],
implementation: {
kind: "registered",
locator: "example.video-pipeline/request-estimate",
@@ -75,56 +84,28 @@ export const videoManifest: ModuleManifest = {
},
},
{
- name: videoProducers.requestGeneration.name,
+ name: videoProducers.requestSeedanceMini.name,
inputs: [
- { name: "narrative", type: videoTypes.narrative },
- { name: "schedule", type: videoTypes.schedule },
+ { name: "narrative", type: contractTypes.narrative },
+ { name: "estimate", type: videoTypes.estimate },
],
outputs: [],
- needs: [{ name: "basis", wants: videoTypes.basis }],
- implementation: {
- kind: "registered",
- locator: "example.video-pipeline/request-generation",
- digest: videoImplementations.requestGeneration,
- },
- },
- {
- name: videoProducers.requestRecognition.name,
- inputs: [{ name: "basis", type: videoTypes.basis }],
- outputs: [],
- needs: [{ name: "evidence", wants: videoTypes.evidence }],
- implementation: {
- kind: "registered",
- locator: "example.video-pipeline/request-recognition",
- digest: videoImplementations.requestRecognition,
- },
- },
- {
- name: videoProducers.locateSpeech.name,
- inputs: [
- { name: "narrative", type: videoTypes.narrative },
- { name: "evidence", type: videoTypes.evidence },
- ],
- outputs: [{ name: "map", type: videoTypes.speechMap }],
- needs: [],
+ needs: [{ name: "media", wants: videoTypes.seedanceMiniMedia }],
implementation: {
kind: "registered",
- locator: "example.video-pipeline/locate-speech",
- digest: videoImplementations.locateSpeech,
+ locator: "example.video-pipeline/request-seedance-mini",
+ digest: videoImplementations.requestSeedanceMini,
},
},
{
- name: videoProducers.temporalizeCaption.name,
- inputs: [
- { name: "narrative", type: videoTypes.narrative },
- { name: "map", type: videoTypes.speechMap },
- ],
- outputs: [{ name: "caption", type: videoTypes.caption }],
+ name: videoProducers.assembleBasis.name,
+ inputs: [{ name: "media", type: videoTypes.seedanceMiniMedia }],
+ outputs: [{ name: "basis", type: contractTypes.speechBasis }],
needs: [],
implementation: {
kind: "registered",
- locator: "example.video-pipeline/temporalize-caption",
- digest: videoImplementations.temporalizeCaption,
+ locator: "example.video-pipeline/assemble-basis",
+ digest: videoImplementations.assembleBasis,
},
},
],
@@ -132,53 +113,78 @@ export const videoManifest: ModuleManifest = {
export const videoPlan: BuildPlan = {
format: "svml.plan@0",
- id: "video-pipeline",
+ id: "explicit-whisperx-video-pipeline",
steps: [
{
id: "request-estimate",
producer: videoProducers.requestEstimate,
inputs: { narrative: "narrative:root" },
outputs: {},
- needs: { schedule: { id: "need:schedule", result: "schedule:root", accepts: "exact" } },
+ needs: { estimate: { id: "need:estimate", result: "estimate:root", accepts: "exact" } },
},
{
- id: "request-generation",
- producer: videoProducers.requestGeneration,
- inputs: { narrative: "narrative:root", schedule: "schedule:root" },
+ id: "request-seedance-mini",
+ producer: videoProducers.requestSeedanceMini,
+ inputs: { narrative: "narrative:root", estimate: "estimate:root" },
outputs: {},
- needs: { basis: { id: "need:basis", result: "basis:root", accepts: "exact" } },
+ needs: { media: { id: "need:seedance-mini", result: "seedance-media:root", accepts: "exact" } },
},
{
- id: "request-recognition",
- producer: videoProducers.requestRecognition,
+ id: "assemble-basis",
+ producer: videoProducers.assembleBasis,
+ inputs: { media: "seedance-media:root" },
+ outputs: { basis: "basis:root" },
+ needs: {},
+ },
+ {
+ id: "request-whisperx",
+ producer: whisperXProducers.request,
inputs: { basis: "basis:root" },
outputs: {},
- needs: { evidence: { id: "need:evidence", result: "evidence:root", accepts: "exact" } },
+ needs: { alignment: { id: "need:whisperx", result: "whisperx:root", accepts: "exact" } },
+ },
+ {
+ id: "normalize-whisperx",
+ producer: whisperXProducers.normalize,
+ inputs: { whisperx: "whisperx:root" },
+ outputs: { evidence: "evidence:root" },
+ needs: {},
},
{
id: "locate-speech",
- producer: videoProducers.locateSpeech,
- inputs: { narrative: "narrative:root", evidence: "evidence:root" },
+ producer: speechAlignProducers.locate,
+ inputs: { narrative: "narrative:root", basis: "basis:root", evidence: "evidence:root" },
outputs: { map: "speech-map:root" },
needs: {},
},
{
id: "temporalize-caption",
- producer: videoProducers.temporalizeCaption,
+ producer: captionProducers.temporalize,
inputs: { narrative: "narrative:root", map: "speech-map:root" },
outputs: { caption: "caption:root" },
needs: {},
},
],
- goals: [{ record: "caption:root", type: videoTypes.caption, accepts: "exact" }],
+ goals: [{ record: "caption:root", type: contractTypes.timedCaptionProjection, accepts: "exact" }],
};
+export const videoClosure = createResolvedClosure([
+ contractsManifest,
+ videoManifest,
+ whisperXManifest,
+ speechAlignManifest,
+ captionManifest,
+]);
+
export function createVideoBuild(): BuildState {
- const closure = createResolvedClosure([videoManifest]);
+ const parsed = parseScript(
+ "pipeline.svml",
+ "<line><that was insane | what the fuck></line>",
+ );
const narrative = sealRecord({
id: "narrative:root",
- type: videoTypes.narrative,
- value: { kind: "inline", value: { speech: "what the fuck", display: "that was insane" } },
+ type: contractTypes.narrative,
+ value: { kind: "inline", value: narrativeValue(parsed) },
conformance: "exact",
origin: {
kind: "authored",
@@ -187,9 +193,11 @@ export function createVideoBuild(): BuildState {
sourceName: "pipeline.svml",
},
});
- return start(link(closure, [sealTypedModule({
+ return start(link(videoClosure, [sealTypedModule({
id: "author:video-pipeline",
- closureDigest: closure.digest,
+ closureDigest: videoClosure.digest,
records: [narrative],
})]), videoPlan);
}
+
+export { contractTypes, whisperXTypes };
diff --git a/packages/driver-node/test/video-pipeline.test.ts b/packages/driver-node/test/video-pipeline.test.ts
index 85144a68..d639f324 100644
--- a/packages/driver-node/test/video-pipeline.test.ts
+++ b/packages/driver-node/test/video-pipeline.test.ts
@@ -1,6 +1,19 @@
import assert from "node:assert/strict";
import test from "node:test";
+import { temporalizeCaption } from "@svml/caption";
+import {
+ sealProgramSpace,
+ sealSpeechBasis,
+} from "@svml/contracts";
+import type {
+ AlignedTranscriptEvidence,
+ CompleteSemanticMap,
+ Narrative,
+ SpeechBasis,
+ TimedCaptionProjection,
+} from "@svml/contracts";
+import { digestOf, start } from "@svml/core";
import {
HostRegistry,
NodeDriver,
@@ -8,121 +21,330 @@ import {
parseBuildState,
serializeBuildState,
} from "@svml/driver-node";
+import { locateSpeechTiming } from "@svml/speech-align";
+import {
+ normalizeWhisperXAlignment,
+ sealWhisperXAlignmentEvidence,
+ whisperXImplementationDigests,
+ whisperXProducers,
+ whisperXRequestForBasis,
+} from "@svml/whisperx";
+import type { WhisperXAlignmentEvidence } from "@svml/whisperx";
import {
+ contractTypes,
createVideoBuild,
videoImplementations,
videoProducers,
videoTypes,
+ whisperXTypes,
} from "./video-pipeline-fixture.js";
+import { captionImplementationDigest, captionProducers } from "@svml/caption";
+import { speechAlignProducers, speechLocatorDigest } from "@svml/speech-align";
function inlineObject(value: unknown): Readonly<Record<string, unknown>> {
if (value === null || Array.isArray(value) || typeof value !== "object") throw new Error("expected object");
return value as Readonly<Record<string, unknown>>;
}
-function inlineString(value: unknown): string {
- if (typeof value !== "string") throw new Error("expected string");
- return value;
-}
-
-function inlineNumber(value: unknown): number {
- if (typeof value !== "number" || !Number.isFinite(value)) throw new Error("expected number");
- return value;
+function inlineValue<T>(value: unknown): T {
+ return value as T;
}
-test("estimate, generation and recognition pause independently without replaying the immutable DAG", async () => {
+test("external methods are fixed by the BuildPlan while Runtime binds only exact execution endpoints", async () => {
const host = new HostRegistry();
const providers = new ProviderRegistry();
- const calls = { estimate: 0, generate: 0, recognize: 0, locate: 0, caption: 0 };
+ const calls = {
+ estimate: 0,
+ seedance: 0,
+ assemble: 0,
+ whisperRequest: 0,
+ whisperNormalize: 0,
+ locate: 0,
+ caption: 0,
+ };
host.registerProducer(videoProducers.requestEstimate, videoImplementations.requestEstimate, ({ inputs }) => {
calls.estimate += 1;
- const narrative = inputs.narrative;
- assert.equal(narrative?.value.kind, "inline");
- return { outputs: {}, needs: { schedule: { speech: inlineString(inlineObject(narrative.value.value).speech) } } };
+ assert.equal(inputs.narrative?.value.kind, "inline");
+ const narrative = inlineValue<Narrative>(inputs.narrative.value.value);
+ return {
+ outputs: {},
+ needs: {
+ estimate: {
+ contract: "example.official-speech-estimate-request@1",
+ speech: narrative.serializations.speech,
+ },
+ },
+ };
+ });
+ host.registerProducer(videoProducers.requestSeedanceMini, videoImplementations.requestSeedanceMini, ({ inputs }) => {
+ calls.seedance += 1;
+ assert.equal(inputs.narrative?.value.kind, "inline");
+ assert.equal(inputs.estimate?.value.kind, "inline");
+ const narrative = inlineValue<Narrative>(inputs.narrative.value.value);
+ const estimate = inlineObject(inputs.estimate.value.value);
+ return {
+ outputs: {},
+ needs: {
+ media: {
+ contract: "example.seedance-mini-speech-request@1",
+ model: "mini",
+ script: narrative.serializations.speech,
+ durationSec: estimate.durationSec as number,
+ },
+ },
+ };
});
- host.registerProducer(videoProducers.requestGeneration, videoImplementations.requestGeneration, ({ inputs }) => {
- calls.generate += 1;
- assert.equal(inputs.schedule?.value.kind, "inline");
- return { outputs: {}, needs: { basis: { durationSec: inlineNumber(inlineObject(inputs.schedule.value.value).durationSec) } } };
+ host.registerProducer(videoProducers.assembleBasis, videoImplementations.assembleBasis, ({ inputs }) => {
+ calls.assemble += 1;
+ assert.equal(inputs.media?.value.kind, "inline");
+ const media = inlineObject(inputs.media.value.value);
+ const durationSec = media.durationSec as number;
+ const programSpace = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec,
+ frameRate: { numerator: 1_000, denominator: 1 },
+ });
+ const audioDigest = media.audioDigest as SpeechBasis["audio"]["digest"];
+ const visualDigest = media.visualDigest as SpeechBasis["audio"]["digest"];
+ const basis = sealSpeechBasis({
+ contract: "svml.speech-basis@1",
+ programSpace,
+ audio: { digest: audioDigest, size: 1, mediaType: "audio/wav", durationSec },
+ visualTrack: { clips: [{
+ segmentId: "line",
+ artifact: { digest: visualDigest, size: 1, mediaType: "video/mp4", durationSec },
+ startSec: 0,
+ endSec: durationSec,
+ }] },
+ segments: [{ segmentId: "line", startSec: 0, endSec: durationSec, sourceArtifactDigest: visualDigest }],
+ });
+ return { outputs: { basis: { kind: "inline", value: basis } }, needs: {} };
});
- host.registerProducer(videoProducers.requestRecognition, videoImplementations.requestRecognition, ({ inputs }) => {
- calls.recognize += 1;
+ host.registerProducer(whisperXProducers.request, whisperXImplementationDigests.request, ({ inputs }) => {
+ calls.whisperRequest += 1;
assert.equal(inputs.basis?.value.kind, "inline");
- return { outputs: {}, needs: { evidence: { audio: inlineString(inlineObject(inputs.basis.value.value).audio) } } };
+ return {
+ outputs: {},
+ needs: { alignment: whisperXRequestForBasis(inlineValue<SpeechBasis>(inputs.basis.value.value), { language: "en" }) },
+ };
});
- host.registerProducer(videoProducers.locateSpeech, videoImplementations.locateSpeech, ({ inputs }) => {
+ host.registerProducer(whisperXProducers.normalize, whisperXImplementationDigests.normalize, ({ inputs }) => {
+ calls.whisperNormalize += 1;
+ assert.equal(inputs.whisperx?.value.kind, "inline");
+ const evidence = normalizeWhisperXAlignment(
+ inlineValue<WhisperXAlignmentEvidence>(inputs.whisperx.value.value),
+ );
+ return { outputs: { evidence: { kind: "inline", value: evidence } }, needs: {} };
+ });
+ host.registerProducer(speechAlignProducers.locate, speechLocatorDigest, ({ inputs }) => {
calls.locate += 1;
+ assert.equal(inputs.narrative?.value.kind, "inline");
+ assert.equal(inputs.basis?.value.kind, "inline");
assert.equal(inputs.evidence?.value.kind, "inline");
- return { outputs: { map: { kind: "inline", value: { startSec: 0.1, endSec: 0.72 } } }, needs: {} };
+ const map = locateSpeechTiming(
+ inlineValue<Narrative>(inputs.narrative.value.value),
+ inlineValue<SpeechBasis>(inputs.basis.value.value),
+ inlineValue<AlignedTranscriptEvidence>(inputs.evidence.value.value),
+ );
+ return { outputs: { map: { kind: "inline", value: map } }, needs: {} };
});
- host.registerProducer(videoProducers.temporalizeCaption, videoImplementations.temporalizeCaption, ({ inputs }) => {
+ host.registerProducer(captionProducers.temporalize, captionImplementationDigest, ({ inputs }) => {
calls.caption += 1;
assert.equal(inputs.narrative?.value.kind, "inline");
assert.equal(inputs.map?.value.kind, "inline");
- const narrative = inlineObject(inputs.narrative.value.value);
- const map = inlineObject(inputs.map.value.value);
- return {
- outputs: {
- caption: {
- kind: "inline",
- value: {
- text: inlineString(narrative.display),
- startSec: inlineNumber(map.startSec),
- endSec: inlineNumber(map.endSec),
- },
- },
- },
- needs: {},
- };
+ const caption = temporalizeCaption(
+ inlineValue<Narrative>(inputs.narrative.value.value),
+ inlineValue<CompleteSemanticMap>(inputs.map.value.value),
+ );
+ return { outputs: { caption: { kind: "inline", value: caption } }, needs: {} };
});
const driver = new NodeDriver({ registry: host, providers });
const atEstimate = await driver.run(createVideoBuild());
assert.equal(atEstimate.blocked[0]?.reason, "missing-provider");
- assert.deepEqual(calls, { estimate: 1, generate: 0, recognize: 0, locate: 0, caption: 0 });
+ assert.match(atEstimate.blocked[0]?.subject ?? "", /OfficialSpeechDurationEstimate/u);
- providers.registerProvider("runtime:estimate", videoTypes.schedule, () => ({
+ providers.registerProvider("runtime:official-estimate", videoTypes.estimate, () => ({
value: { kind: "inline", value: { durationSec: 1 } },
conformance: "exact",
delivery: "executed",
metadata: {},
}));
- const atGeneration = await driver.run(parseBuildState(serializeBuildState(atEstimate.state)));
- assert.equal(atGeneration.blocked[0]?.reason, "missing-provider");
- assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 0, locate: 0, caption: 0 });
+ const atSeedance = await driver.run(parseBuildState(serializeBuildState(atEstimate.state)));
+ assert.equal(atSeedance.blocked[0]?.reason, "missing-provider");
+ assert.match(atSeedance.blocked[0]?.subject ?? "", /SeedanceMiniSpeechMedia/u);
- providers.registerProvider("runtime:seedance-mini", videoTypes.basis, ({ need }) => {
- assert.deepEqual(need.constraints, { durationSec: 1 });
+ providers.registerProvider("runtime:kie-seedance-mini", videoTypes.seedanceMiniMedia, ({ need }) => {
+ assert.deepEqual(need.constraints, {
+ contract: "example.seedance-mini-speech-request@1",
+ durationSec: 1,
+ model: "mini",
+ script: "what the fuck",
+ });
return {
- value: { kind: "inline", value: { audio: "artifact:audio", visual: "artifact:visual" } },
+ value: { kind: "inline", value: {
+ model: "mini",
+ audioDigest: digestOf("artifact:audio"),
+ visualDigest: digestOf("artifact:visual"),
+ durationSec: 1,
+ } },
conformance: "exact",
delivery: "executed",
- metadata: { model: "mini" },
+ metadata: { provider: "kie", model: "seedance-mini" },
};
});
- const atRecognition = await driver.run(parseBuildState(serializeBuildState(atGeneration.state)));
- assert.equal(atRecognition.blocked[0]?.reason, "missing-provider");
- assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 0, caption: 0 });
+ // A generic Evidence provider cannot satisfy the explicit WhisperX Need.
+ providers.registerProvider("runtime:generic-stt", contractTypes.alignedTranscriptEvidence, () => {
+ throw new Error("generic STT must never run for an explicit WhisperX Need");
+ });
+ const atWhisperX = await driver.run(parseBuildState(serializeBuildState(atSeedance.state)));
+ assert.equal(atWhisperX.blocked[0]?.reason, "missing-provider");
+ assert.match(atWhisperX.blocked[0]?.subject ?? "", /WhisperXAlignmentEvidence/u);
- providers.registerProvider("runtime:whisperx", videoTypes.evidence, () => ({
- value: { kind: "inline", value: { transcript: "what the fuck" } },
- conformance: "exact",
- delivery: "executed",
- metadata: { adapter: "whisperx" },
- }));
- const completed = await driver.run(parseBuildState(serializeBuildState(atRecognition.state)));
+ providers.registerProvider("runtime:whisperx-local", whisperXTypes.alignmentEvidence, ({ need }) => {
+ const request = inlineObject(need.constraints);
+ const audio = inlineObject(request.audio);
+ return {
+ value: {
+ kind: "inline",
+ value: sealWhisperXAlignmentEvidence({
+ contract: "svml.whisperx-alignment-evidence@1",
+ engine: "whisperx",
+ basisDigest: request.basisDigest as WhisperXAlignmentEvidence["basisDigest"],
+ audioArtifactDigest: audio.digest as WhisperXAlignmentEvidence["audioArtifactDigest"],
+ programSpaceDigest: request.programSpaceDigest as WhisperXAlignmentEvidence["programSpaceDigest"],
+ rawEvidenceArtifactDigest: digestOf("artifact:whisperx-json"),
+ durationSec: 1,
+ segments: [{
+ sourceSegmentId: "line",
+ startSec: 0,
+ endSec: 1,
+ words: [
+ { text: "what", startSec: 0.1, endSec: 0.25, score: 0.98 },
+ { text: "the", startSec: 0.3, endSec: 0.42, score: 0.98 },
+ { text: "fuck", startSec: 0.48, endSec: 0.72, score: 0.97 },
+ ],
+ chars: [],
+ }],
+ }),
+ },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: { adapter: "whisperx.local", model: "fixture" },
+ };
+ });
+ const completed = await driver.run(parseBuildState(serializeBuildState(atWhisperX.state)));
assert.equal(completed.status, "complete");
- assert.deepEqual(calls, { estimate: 1, generate: 1, recognize: 1, locate: 1, caption: 1 });
+ assert.deepEqual(calls, {
+ estimate: 1,
+ seedance: 1,
+ assemble: 1,
+ whisperRequest: 1,
+ whisperNormalize: 1,
+ locate: 1,
+ caption: 1,
+ });
assert.deepEqual(completed.state.receipts.map((receipt) => receipt.fulfiller), [
- "runtime:estimate",
- "runtime:seedance-mini",
- "runtime:whisperx",
+ "runtime:official-estimate",
+ "runtime:kie-seedance-mini",
+ "runtime:whisperx-local",
]);
+ const caption = completed.state.records.find((record) => record.id === "caption:root");
+ assert.equal(caption?.value.kind, "inline");
+ const value = inlineValue<TimedCaptionProjection>(caption?.value.kind === "inline" ? caption.value.value : {});
assert.deepEqual(
- completed.state.records.find((record) => record.id === "caption:root")?.value,
- { kind: "inline", value: { text: "that was insane", startSec: 0.1, endSec: 0.72 } },
+ [value.text, value.regions[0]?.display, value.regions[0]?.startSec, value.regions[0]?.endSec],
+ ["that was insane", "that was insane", 0.1, 0.72],
);
});
+
+test("an exact Provider result cannot wash a substitute input back to exact", async () => {
+ const initial = createVideoBuild();
+ const plan = {
+ ...initial.plan,
+ steps: initial.plan.steps.map((step) => {
+ if (step.id === "request-estimate") {
+ return { ...step, needs: { estimate: { ...step.needs.estimate!, accepts: "substitute" as const } } };
+ }
+ if (step.id === "request-seedance-mini") {
+ return { ...step, needs: { media: { ...step.needs.media!, accepts: "substitute" as const } } };
+ }
+ return step;
+ }),
+ goals: initial.plan.goals.map((goal) => ({ ...goal, accepts: "substitute" as const })),
+ };
+ const host = new HostRegistry();
+ host.registerProducer(videoProducers.requestEstimate, videoImplementations.requestEstimate, () => ({
+ outputs: {},
+ needs: { estimate: { speech: "what the fuck" } },
+ }));
+ host.registerProducer(videoProducers.requestSeedanceMini, videoImplementations.requestSeedanceMini, () => ({
+ outputs: {},
+ needs: { media: { model: "mini" } },
+ }));
+ const providers = new ProviderRegistry();
+ providers.registerProvider("runtime:estimated-duration", videoTypes.estimate, () => ({
+ value: { kind: "inline", value: { durationSec: 1 } },
+ conformance: "substitute",
+ delivery: "provided",
+ metadata: {},
+ }));
+ providers.registerProvider("runtime:exact-seedance", videoTypes.seedanceMiniMedia, () => ({
+ value: { kind: "inline", value: {
+ model: "mini",
+ audioDigest: digestOf("floor:audio"),
+ visualDigest: digestOf("floor:visual"),
+ durationSec: 1,
+ } },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ }));
+
+ const result = await new NodeDriver({ registry: host, providers }).run(start(initial.program, plan));
+ assert.equal(result.status, "paused", "the unregistered Basis assembler should pause the test after both Needs");
+ const seedanceNeed = result.state.needs.find((need) => need.id === "need:seedance-mini");
+ const seedanceReceipt = result.state.receipts.find((receipt) => receipt.need === "need:seedance-mini");
+ const media = result.state.records.find((record) => record.id === "seedance-media:root");
+ assert.equal(seedanceNeed?.conformanceFloor, "substitute");
+ assert.equal(seedanceReceipt?.fulfillmentConformance, "exact");
+ assert.equal(seedanceReceipt?.conformance, "substitute");
+ assert.equal(media?.conformance, "substitute");
+});
+
+test("Core rejects an impossible exact downstream Need before an expensive Provider can run", async () => {
+ const initial = createVideoBuild();
+ const plan = {
+ ...initial.plan,
+ steps: initial.plan.steps.map((step) => step.id === "request-estimate"
+ ? { ...step, needs: { estimate: { ...step.needs.estimate!, accepts: "substitute" as const } } }
+ : step),
+ };
+ const host = new HostRegistry();
+ host.registerProducer(videoProducers.requestEstimate, videoImplementations.requestEstimate, () => ({
+ outputs: {}, needs: { estimate: { speech: "what the fuck" } },
+ }));
+ host.registerProducer(videoProducers.requestSeedanceMini, videoImplementations.requestSeedanceMini, () => ({
+ outputs: {}, needs: { media: { model: "mini" } },
+ }));
+ const providers = new ProviderRegistry();
+ providers.registerProvider("runtime:estimated-duration", videoTypes.estimate, () => ({
+ value: { kind: "inline", value: { durationSec: 1 } },
+ conformance: "substitute",
+ delivery: "provided",
+ metadata: {},
+ }));
+ let expensiveCalls = 0;
+ providers.registerProvider("runtime:paid-seedance", videoTypes.seedanceMiniMedia, () => {
+ expensiveCalls += 1;
+ throw new Error("must not run");
+ });
+
+ const result = await new NodeDriver({ registry: host, providers }).run(start(initial.program, plan));
+ assert.equal(result.status, "failed");
+ assert.equal(result.state.diagnostics.at(-1)?.code, "CONFORMANCE_FLOOR_UNSATISFIABLE");
+ assert.equal(expensiveCalls, 0);
+});
diff --git a/packages/protocol/src/build.ts b/packages/protocol/src/build.ts
index d0813445..e715ac31 100644
--- a/packages/protocol/src/build.ts
+++ b/packages/protocol/src/build.ts
@@ -67,6 +67,8 @@ export type Need = {
readonly requestedBy: DerivationId;
readonly result: RecordId;
readonly accepts: NeedAcceptance;
+ /** Worst conformance inherited from the Producer inputs that requested this Need. */
+ readonly conformanceFloor: Conformance;
readonly requestDigest: Digest;
};
@@ -75,11 +77,28 @@ export type Receipt = {
readonly need: NeedId;
readonly requestDigest: Digest;
readonly fulfiller: string;
+ /** Conformance reported by the external fulfiller before upstream quality is applied. */
+ readonly fulfillmentConformance: Conformance;
+ /** Effective conformance after applying the Need's inherited floor. */
readonly conformance: Conformance;
readonly delivery: Delivery;
readonly output: RecordId;
readonly outputDigest: Digest;
readonly metadata: CanonicalValue;
+ readonly event: {
+ readonly id: EventId;
+ readonly digest: Digest;
+ };
+};
+
+export type RecordDigestBinding = {
+ readonly id: RecordId;
+ readonly digest: Digest;
+};
+
+export type NeedDigestBinding = {
+ readonly id: NeedId;
+ readonly requestDigest: Digest;
};
export type Derivation = {
@@ -87,9 +106,13 @@ export type Derivation = {
readonly step: StepId;
readonly producer: ProducerRef;
readonly implementationDigest: Digest;
- readonly inputs: readonly RecordId[];
- readonly outputs: readonly RecordId[];
- readonly needs: readonly NeedId[];
+ readonly inputs: readonly RecordDigestBinding[];
+ readonly outputs: readonly RecordDigestBinding[];
+ readonly needs: readonly NeedDigestBinding[];
+ readonly event: {
+ readonly id: EventId;
+ readonly digest: Digest;
+ };
};
export type NeedBinding = {
diff --git a/packages/script/README.md b/packages/script/README.md
index ad4d4705..71960365 100644
--- a/packages/script/README.md
+++ b/packages/script/README.md
@@ -27,6 +27,7 @@ not indentation: the compact spelling
`<opening><ALICE>我先说。<BOB>我回答。</opening>` has the same semantic value.
The package exports its Manifest, `parseScript`, semantic/source-map projection helpers, a
-semantic-preserving formatter and the raw `decodeScriptSurface` handler. The first bootstrap keeps
-the Narrative contract in this package; it can be split into a domain-contract package later
-without changing Text or Core.
+semantic-preserving formatter and the raw `decodeScriptSurface` handler. Source ranges and parser
+state remain private to Script; its authored Narrative Record uses the Frontend-neutral type from
+`@svml/contracts`, so third-party author surfaces can feed the same WhisperX, locator and caption
+components without importing Script internals.
diff --git a/packages/script/package.json b/packages/script/package.json
index 2fcf19cf..05d4ea7c 100644
--- a/packages/script/package.json
+++ b/packages/script/package.json
@@ -7,6 +7,7 @@
".": "./src/index.ts"
},
"dependencies": {
+ "@svml/contracts": "workspace:*",
"@svml/core": "workspace:*",
"@svml/protocol": "workspace:*"
}
diff --git a/packages/script/src/manifest.ts b/packages/script/src/manifest.ts
index bf84d853..5ca34556 100644
--- a/packages/script/src/manifest.ts
+++ b/packages/script/src/manifest.ts
@@ -1,233 +1,24 @@
+import {
+ contractTypes,
+ contractsManifestDigest,
+ contractsModuleRef,
+ narrativeSchema,
+} from "@svml/contracts";
import { digestOf } from "@svml/core";
-import type { ModuleManifest, TypeRef, ValueSchema } from "@svml/protocol";
+import type { ModuleManifest, TypeRef } from "@svml/protocol";
export const scriptModuleRef = { name: "@svml/script", version: "0.0.0-dev" } as const;
+export const narrativeType: TypeRef = contractTypes.narrative;
+export { narrativeSchema };
-export const narrativeType: TypeRef = {
- module: scriptModuleRef,
- name: "Narrative",
-};
-
-export const scriptSurfaceImplementationDigest = digestOf("@svml/script/surface@0");
-
-const stringSchema = { kind: "string", minLength: 1 } as const;
-const integerSchema = { kind: "number", integer: true, minimum: 0 } as const;
-const boundarySchema: ValueSchema = {
- kind: "object",
- fields: {
- tokenIndex: { schema: integerSchema },
- structuralPosition: { schema: integerSchema },
- segmentId: { schema: stringSchema, optional: true },
- },
-};
-const edgeSchema: ValueSchema = {
- kind: "object",
- fields: {
- affinity: { schema: { kind: "string", enum: ["left", "right"] } },
- boundary: { schema: boundarySchema },
- },
-};
-
-export const narrativeSchema: ValueSchema = {
- kind: "object",
- fields: {
- contract: { schema: { kind: "literal", value: "svml.narrative@0" } },
- segments: {
- schema: {
- kind: "array",
- minItems: 1,
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- index: { schema: integerSchema },
- startAnchorId: { schema: stringSchema },
- endAnchorId: { schema: stringSchema },
- tokenStart: { schema: integerSchema },
- tokenEndExclusive: { schema: integerSchema },
- },
- },
- },
- },
- tokens: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- index: { schema: integerSchema },
- segmentId: { schema: stringSchema },
- segmentTokenIndex: { schema: integerSchema },
- startAnchorId: { schema: stringSchema },
- endAnchorId: { schema: stringSchema },
- text: { schema: stringSchema },
- normalized: { schema: stringSchema },
- },
- },
- },
- },
- turns: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- segmentId: { schema: stringSchema },
- role: { schema: stringSchema, optional: true },
- tokenStart: { schema: integerSchema },
- tokenEndExclusive: { schema: integerSchema },
- },
- },
- },
- },
- selections: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- occurrences: {
- schema: {
- kind: "array",
- minItems: 1,
- items: {
- kind: "object",
- fields: {
- occurrence: { schema: integerSchema },
- open: { schema: edgeSchema },
- close: { schema: edgeSchema },
- },
- },
- },
- },
- },
- },
- },
- },
- moments: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- occurrences: {
- schema: {
- kind: "array",
- minItems: 1,
- items: {
- kind: "object",
- fields: {
- occurrence: { schema: integerSchema },
- affinity: { schema: { kind: "string", enum: ["left", "right"] } },
- boundary: { schema: boundarySchema },
- },
- },
- },
- },
- },
- },
- },
- },
- captionProjection: {
- schema: {
- kind: "object",
- fields: {
- contract: { schema: { kind: "literal", value: "svml.caption-projection@0" } },
- text: { schema: { kind: "string" } },
- regions: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- display: { schema: { kind: "string" } },
- segmentId: { schema: stringSchema },
- startToken: { schema: integerSchema },
- endTokenExclusive: { schema: integerSchema },
- kind: { schema: { kind: "string", enum: ["identity", "alias", "hidden"] } },
- refinements: {
- schema: {
- kind: "array",
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- display: { schema: stringSchema },
- displayStart: { schema: integerSchema },
- displayEnd: { schema: integerSchema },
- startToken: { schema: integerSchema },
- endTokenExclusive: { schema: integerSchema },
- relation: { schema: { kind: "literal", value: "exact" } },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- semanticIndex: {
- schema: {
- kind: "object",
- fields: {
- contract: { schema: { kind: "literal", value: "svml.semantic-index@0" } },
- anchors: {
- schema: {
- kind: "array",
- minItems: 2,
- items: {
- kind: "object",
- fields: {
- id: { schema: stringSchema },
- kind: {
- schema: {
- kind: "string",
- enum: ["segment-start", "token-start", "token-end", "segment-end"],
- },
- },
- segmentId: { schema: stringSchema },
- tokenId: { schema: stringSchema, optional: true },
- segmentTokenIndex: { schema: integerSchema, optional: true },
- },
- },
- },
- },
- digest: { schema: stringSchema },
- },
- },
- },
- serializations: {
- schema: {
- kind: "object",
- fields: {
- dialogue: { schema: { kind: "string" } },
- speech: { schema: { kind: "string" } },
- },
- },
- },
- },
-};
+export const scriptSurfaceImplementationDigest = digestOf("@svml/script/surface@1");
export const scriptManifest: ModuleManifest = {
format: "svml.module@0",
name: scriptModuleRef.name,
version: scriptModuleRef.version,
- dependencies: [],
- types: [
- {
- name: narrativeType.name,
- schema: narrativeSchema,
- description: "Authoritative narrative, pure serializations, Caption Projection and 2M + 2N identities.",
- },
- ],
+ dependencies: [{ module: contractsModuleRef, digest: contractsManifestDigest }],
+ types: [],
surfaces: [
{
name: "script",
diff --git a/packages/script/src/parser.ts b/packages/script/src/parser.ts
index 43b747fe..a6d962d2 100644
--- a/packages/script/src/parser.ts
+++ b/packages/script/src/parser.ts
@@ -694,6 +694,7 @@ export function parseScript(
};
return {
+ contract: "svml.narrative@0",
segments,
tokens,
turns,
diff --git a/packages/script/src/types.ts b/packages/script/src/types.ts
index f96ae2d5..0e5cbae6 100644
--- a/packages/script/src/types.ts
+++ b/packages/script/src/types.ts
@@ -1,12 +1,28 @@
+import type {
+ Affinity,
+ CaptionProjection,
+ CaptionRefinement,
+ CaptionRegion,
+ MarkerBoundary,
+ Narrative,
+ NarrativeMoment,
+ NarrativeMomentOccurrence,
+ NarrativeSegment,
+ NarrativeSelection,
+ NarrativeSelectionOccurrence,
+ NarrativeToken,
+ NarrativeTurn,
+ SemanticAnchor,
+} from "@svml/contracts";
import type { CanonicalValue, SourceRange, StoredValue, TypeRef } from "@svml/protocol";
-export type Affinity = "left" | "right";
-
-export type MarkerBoundary = {
- readonly tokenIndex: number;
- readonly structuralPosition: number;
- readonly segmentId?: string;
-};
+export type {
+ Affinity,
+ CaptionProjection,
+ MarkerBoundary,
+ Narrative,
+ SemanticAnchor,
+} from "@svml/contracts";
export type ParsedTextAtom = {
readonly kind: "text";
@@ -25,120 +41,43 @@ export type ParsedRoleAtom = {
export type ParsedAtom = ParsedTextAtom | ParsedRoleAtom;
-export type ParsedSegment = {
- readonly id: string;
- readonly index: number;
- readonly startAnchorId: string;
- readonly endAnchorId: string;
- readonly tokenStart: number;
- readonly tokenEndExclusive: number;
+export type ParsedSegment = NarrativeSegment & {
readonly atoms: readonly ParsedAtom[];
readonly range: SourceRange;
readonly selfClosing: boolean;
};
-export type ParsedToken = {
- readonly id: string;
- readonly index: number;
- readonly segmentId: string;
- readonly segmentTokenIndex: number;
- readonly startAnchorId: string;
- readonly endAnchorId: string;
- readonly text: string;
- readonly normalized: string;
- readonly range: SourceRange;
-};
+export type ParsedToken = NarrativeToken & { readonly range: SourceRange };
+export type ParsedTurn = NarrativeTurn & { readonly range: SourceRange };
-export type ParsedTurn = {
- readonly id: string;
- readonly segmentId: string;
- readonly role?: string;
- readonly tokenStart: number;
- readonly tokenEndExclusive: number;
- readonly range: SourceRange;
-};
-
-export type ParsedSelectionOccurrence = {
- readonly occurrence: number;
- readonly open: {
- readonly affinity: Affinity;
- readonly boundary: MarkerBoundary;
- readonly range: SourceRange;
- };
- readonly close: {
- readonly affinity: Affinity;
- readonly boundary: MarkerBoundary;
- readonly range: SourceRange;
- };
+export type ParsedSelectionOccurrence = NarrativeSelectionOccurrence & {
+ readonly open: NarrativeSelectionOccurrence["open"] & { readonly range: SourceRange };
+ readonly close: NarrativeSelectionOccurrence["close"] & { readonly range: SourceRange };
};
-export type ParsedSelection = {
- readonly id: string;
+export type ParsedSelection = Omit<NarrativeSelection, "occurrences"> & {
readonly occurrences: readonly ParsedSelectionOccurrence[];
};
-export type ParsedMomentOccurrence = {
- readonly occurrence: number;
- readonly affinity: Affinity;
- readonly boundary: MarkerBoundary;
- readonly range: SourceRange;
-};
-
-export type ParsedMoment = {
- readonly id: string;
+export type ParsedMomentOccurrence = NarrativeMomentOccurrence & { readonly range: SourceRange };
+export type ParsedMoment = Omit<NarrativeMoment, "occurrences"> & {
readonly occurrences: readonly ParsedMomentOccurrence[];
};
-export type ParsedCaptionRefinement = {
- readonly id: string;
- readonly display: string;
- readonly displayStart: number;
- readonly displayEnd: number;
- readonly startToken: number;
- readonly endTokenExclusive: number;
- readonly relation: "exact";
-};
-
-export type ParsedCaptionRegion = {
- readonly id: string;
- readonly display: string;
- readonly segmentId: string;
- readonly startToken: number;
- readonly endTokenExclusive: number;
- readonly kind: "identity" | "alias" | "hidden";
- readonly refinements: readonly ParsedCaptionRefinement[];
- readonly range: SourceRange;
-};
-
-export type CaptionProjection = {
- readonly contract: "svml.caption-projection@0";
- readonly text: string;
- readonly regions: readonly ParsedCaptionRegion[];
-};
-
-export type SemanticAnchor = {
- readonly id: string;
- readonly kind: "segment-start" | "token-start" | "token-end" | "segment-end";
- readonly segmentId: string;
- readonly tokenId?: string;
- readonly segmentTokenIndex?: number;
-};
+export type ParsedCaptionRefinement = CaptionRefinement;
+export type ParsedCaptionRegion = CaptionRegion & { readonly range: SourceRange };
-export type ParsedNarrative = {
+export type ParsedNarrative = Omit<
+ Narrative,
+ "segments" | "tokens" | "turns" | "selections" | "moments" | "captionProjection"
+> & {
readonly segments: readonly ParsedSegment[];
readonly tokens: readonly ParsedToken[];
readonly turns: readonly ParsedTurn[];
readonly selections: readonly ParsedSelection[];
readonly moments: readonly ParsedMoment[];
- readonly captionProjection: CaptionProjection;
- readonly semanticIndex: {
- readonly contract: "svml.semantic-index@0";
- readonly anchors: readonly SemanticAnchor[];
- readonly digest: string;
- };
- readonly serializations: {
- readonly dialogue: string;
- readonly speech: string;
+ readonly captionProjection: Omit<CaptionProjection, "regions"> & {
+ readonly regions: readonly ParsedCaptionRegion[];
};
};
diff --git a/packages/speech-align/README.md b/packages/speech-align/README.md
index 0051fbad..208cf9c6 100644
--- a/packages/speech-align/README.md
+++ b/packages/speech-align/README.md
@@ -7,9 +7,15 @@ word, character, score, speech-activity and Segment evidence used here.
The locator aligns each known Script Segment independently. A bounded monotonic M:N dynamic
program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
-Within a selected group, timed evidence characters determine Script token boundaries; word times,
-speech-activity bounds and deterministic zero-width estimates are fallbacks in that order. The result contains
-one timing for every Script token and all `2M + 2N` semantic anchors.
+Within a selected group, timed evidence characters determine Script token boundaries; word times
+and speech-activity bounds are fallbacks. Missing token runs receive continuous weighted windows
+between their measured neighbors instead of invented point timestamps. The result is quantized once
+into the selected ProgramSpace and contains one timing for every Script token and all `2M + 2N`
+semantic anchors.
+
+The locator accepts Narrative, SpeechBasis and AlignedTranscriptEvidence together. It rejects
+Evidence unless its Basis, audio Artifact and ProgramSpace digests match exactly, even when another
+audio file has the same duration and Segment ids.
Caption display is outside this package. `@svml/caption` composes the resulting complete speech
map with Script's authored Caption Projection and keeps presentation estimates local to the track.
diff --git a/packages/speech-align/package.json b/packages/speech-align/package.json
index 3466e5b1..f350801c 100644
--- a/packages/speech-align/package.json
+++ b/packages/speech-align/package.json
@@ -7,7 +7,11 @@
".": "./src/index.ts"
},
"dependencies": {
+ "@svml/contracts": "workspace:*",
"@svml/core": "workspace:*",
+ "@svml/protocol": "workspace:*"
+ },
+ "devDependencies": {
"@svml/script": "workspace:*"
}
}
diff --git a/packages/speech-align/src/align.ts b/packages/speech-align/src/align.ts
index ba7f9563..c30c62c0 100644
--- a/packages/speech-align/src/align.ts
+++ b/packages/speech-align/src/align.ts
@@ -1,11 +1,6 @@
-import type { ParsedToken } from "@svml/script";
+import type { AlignmentGroup, AlignmentRelation, NarrativeToken, SpeechWordEvidence } from "@svml/contracts";
import { editDistance, normalizeForAlignment } from "./normalize.js";
-import type {
- AlignmentGroup,
- AlignmentRelation,
- SpeechWordEvidence,
-} from "./types.js";
const EPSILON = 1e-9;
const DEFAULT_MAX_GROUP = 4;
@@ -35,7 +30,7 @@ function evidenceReliability(words: readonly SpeechWordEvidence[]): number {
}
function exactWordLcs(
- source: readonly ParsedToken[],
+ source: readonly NarrativeToken[],
evidence: readonly SpeechWordEvidence[],
): number {
const rows = Array.from({ length: source.length + 1 }, () =>
@@ -64,7 +59,7 @@ function relation(
}
function pairedCost(
- source: readonly ParsedToken[],
+ source: readonly NarrativeToken[],
evidence: readonly SpeechWordEvidence[],
): number {
const sourceText = comparisonText(source);
@@ -108,7 +103,7 @@ function extend(
export function alignWordGroups(
sourceSegmentId: string,
- source: readonly ParsedToken[],
+ source: readonly NarrativeToken[],
evidence: readonly SpeechWordEvidence[],
maxGroupSize = DEFAULT_MAX_GROUP,
): AlignmentGroup[] {
diff --git a/packages/speech-align/src/index.ts b/packages/speech-align/src/index.ts
index d13913e8..cc48dd9c 100644
--- a/packages/speech-align/src/index.ts
+++ b/packages/speech-align/src/index.ts
@@ -1,5 +1,6 @@
export { alignWordGroups } from "./align.js";
export { SpeechAlignmentError } from "./error.js";
-export { locateSpeechTiming } from "./locate.js";
+export { locateSpeechTiming, speechLocatorDigest } from "./locate.js";
+export { speechAlignManifest, speechAlignModuleRef, speechAlignProducers } from "./manifest.js";
export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";
export type * from "./types.js";
diff --git a/packages/speech-align/src/locate.ts b/packages/speech-align/src/locate.ts
index 030aab6b..8be6c2ae 100644
--- a/packages/speech-align/src/locate.ts
+++ b/packages/speech-align/src/locate.ts
@@ -1,23 +1,31 @@
-import { digestOf } from "@svml/core";
-import type { ParsedNarrative, ParsedToken } from "@svml/script";
-
-import { alignWordGroups } from "./align.js";
-import { SpeechAlignmentError } from "./error.js";
-import { alignCharacters, alignmentCharacters } from "./normalize.js";
+import { digestOf, isDigest } from "@svml/core";
import type {
+ AlignedTranscriptEvidence,
+ AlignedTranscriptSegment,
AlignmentGroup,
- CompleteSpeechTimeMap,
+ CompleteSemanticMap,
+ Narrative,
+ NarrativeToken,
SemanticTimePoint,
+ SpeechBasis,
+ SpeechCharacterEvidence,
+ SpeechWordEvidence,
TimedSpeechSegment,
TimedSpeechToken,
TimingQuality,
- AlignedTranscriptEvidence,
- AlignedTranscriptSegment,
- SpeechCharacterEvidence,
- SpeechWordEvidence,
-} from "./types.js";
+} from "@svml/contracts";
+import {
+ computeAlignedTranscriptEvidenceDigest,
+ computeProgramSpaceDigest,
+ computeSpeechBasisDigest,
+} from "@svml/contracts";
+
+import { alignWordGroups } from "./align.js";
+import { SpeechAlignmentError } from "./error.js";
+import { alignCharacters, alignmentCharacters } from "./normalize.js";
const EPSILON = 1e-6;
+export const speechLocatorDigest = digestOf("@svml/speech-align/locate@1");
type MutableTiming = {
startSec: number;
@@ -47,13 +55,81 @@ function validateWindow(start: number, end: number, limit: number, label: string
}
}
-function validateEvidence(narrative: ParsedNarrative, evidence: AlignedTranscriptEvidence): void {
- if (evidence.contract !== "svml.aligned-transcript-evidence@0") {
+function validateBasis(narrative: Narrative, basis: SpeechBasis): void {
+ if (basis.contract !== "svml.speech-basis@1") fail("SPEECH_BASIS_CONTRACT", "Unsupported SpeechBasis contract.");
+ const { basisDigest: _basisDigest, ...basisContent } = basis;
+ if (!isDigest(basis.basisDigest) || basis.basisDigest !== computeSpeechBasisDigest(basisContent)) {
+ fail("SPEECH_BASIS_DIGEST", "SpeechBasis digest does not match its canonical contents.");
+ }
+ const { digest: _programDigest, ...programContent } = basis.programSpace;
+ if (!isDigest(basis.programSpace.digest) || basis.programSpace.digest !== computeProgramSpaceDigest(programContent)) {
+ fail("SPEECH_PROGRAM_DIGEST", "ProgramSpace digest does not match its canonical contents.");
+ }
+ const { numerator, denominator } = basis.programSpace.frameRate;
+ if (!Number.isSafeInteger(numerator) || numerator <= 0 || !Number.isSafeInteger(denominator) || denominator <= 0) {
+ fail("SPEECH_FRAME_RATE", "ProgramSpace frame rate must be a positive rational number.");
+ }
+ if (
+ !Number.isFinite(basis.programSpace.durationSec)
+ || basis.programSpace.durationSec <= 0
+ || Math.abs(basis.audio.durationSec - basis.programSpace.durationSec) > EPSILON
+ ) {
+ fail("SPEECH_BASIS_DURATION", "SpeechBasis audio and ProgramSpace must have the same positive duration.");
+ }
+ if (!isDigest(basis.audio.digest)) fail("SPEECH_AUDIO_DIGEST", "SpeechBasis audio digest is invalid.");
+ if (basis.segments.length !== narrative.segments.length) {
+ fail("SPEECH_BASIS_SEGMENTS", "SpeechBasis must cover every Narrative Segment exactly once.");
+ }
+ let previousEnd = 0;
+ for (const [index, segment] of basis.segments.entries()) {
+ const expected = narrative.segments[index]!;
+ if (segment.segmentId !== expected.id) {
+ fail("SPEECH_BASIS_SEGMENTS", `SpeechBasis Segment ${segment.segmentId} does not match ${expected.id}.`);
+ }
+ validateWindow(segment.startSec, segment.endSec, basis.programSpace.durationSec, `Basis Segment ${segment.segmentId}`);
+ if (segment.startSec < previousEnd - EPSILON) {
+ fail("SPEECH_BASIS_SEGMENTS", `SpeechBasis Segment ${segment.segmentId} overlaps its predecessor.`);
+ }
+ if (!isDigest(segment.sourceArtifactDigest)) {
+ fail("SPEECH_BASIS_ARTIFACT", `SpeechBasis Segment ${segment.segmentId} artifact digest is invalid.`);
+ }
+ previousEnd = segment.endSec;
+ }
+}
+
+function validateEvidence(
+ narrative: Narrative,
+ basis: SpeechBasis,
+ evidence: AlignedTranscriptEvidence,
+): void {
+ if (evidence.contract !== "svml.aligned-transcript-evidence@1") {
fail("SPEECH_CONTRACT", `Unsupported aligned-transcript contract ${evidence.contract}.`);
}
if (!Number.isFinite(evidence.durationSec) || evidence.durationSec <= 0) {
fail("SPEECH_DURATION", "Aligned-transcript duration must be positive and finite.");
}
+ if (evidence.basisDigest !== basis.basisDigest) {
+ fail("SPEECH_EVIDENCE_BASIS", "Aligned transcript was measured from a different SpeechBasis.");
+ }
+ if (evidence.audioArtifactDigest !== basis.audio.digest) {
+ fail("SPEECH_EVIDENCE_AUDIO", "Aligned transcript was measured from a different audio Artifact.");
+ }
+ if (evidence.programSpaceDigest !== basis.programSpace.digest) {
+ fail("SPEECH_EVIDENCE_PROGRAM", "Aligned transcript uses a different ProgramSpace.");
+ }
+ if (Math.abs(evidence.durationSec - basis.programSpace.durationSec) > EPSILON) {
+ fail("SPEECH_EVIDENCE_DURATION", "Aligned transcript duration differs from SpeechBasis.");
+ }
+ if (!isDigest(evidence.rawEvidenceArtifactDigest)) {
+ fail("SPEECH_EVIDENCE_RAW", "Aligned transcript raw Artifact digest is invalid.");
+ }
+ const { evidenceDigest: _evidenceDigest, ...evidenceContent } = evidence;
+ if (
+ !isDigest(evidence.evidenceDigest)
+ || evidence.evidenceDigest !== computeAlignedTranscriptEvidenceDigest(evidenceContent)
+ ) {
+ fail("SPEECH_EVIDENCE_DIGEST", "Aligned transcript digest does not match its canonical contents.");
+ }
const expected = new Set(narrative.segments.map((segment) => segment.id));
const seen = new Set<string>();
for (const segment of evidence.segments) {
@@ -130,6 +206,13 @@ function validateEvidence(narrative: ParsedNarrative, evidence: AlignedTranscrip
let previousSegmentEnd = -Infinity;
for (const sourceSegment of narrative.segments) {
const segment = segmentsById.get(sourceSegment.id)!;
+ const basisSegment = basis.segments.find((item) => item.segmentId === sourceSegment.id)!;
+ if (
+ Math.abs(segment.startSec - basisSegment.startSec) > EPSILON
+ || Math.abs(segment.endSec - basisSegment.endSec) > EPSILON
+ ) {
+ fail("SPEECH_EVIDENCE_SEGMENT_AFFINITY", `Segment ${segment.sourceSegmentId} differs from SpeechBasis.`);
+ }
if (
segment.startSec < previousSegmentStart - EPSILON
|| segment.endSec < previousSegmentEnd - EPSILON
@@ -232,7 +315,7 @@ function groupWindow(
function locatePairedGroup(
group: AlignmentGroup,
- source: readonly ParsedToken[],
+ source: readonly NarrativeToken[],
words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
output: Array<MutableTiming | undefined>,
@@ -285,6 +368,7 @@ function speechBounds(segment: AlignedTranscriptSegment): { readonly start: numb
function fillEstimated(
values: Array<MutableTiming | undefined>,
+ source: readonly NarrativeToken[],
startBound: number,
endBound: number,
): MutableTiming[] {
@@ -300,11 +384,19 @@ function fillEstimated(
const left = runStart > 0 ? values[runStart - 1]!.endSec : startBound;
const right = runEnd < values.length ? values[runEnd]!.startSec : endBound;
const usableRight = Math.max(left, right);
+ const weights = source
+ .slice(runStart, runEnd)
+ .map((token) => Math.max(1, [...token.normalized].length));
+ const totalWeight = weights.reduce((sum, weight) => sum + weight, 0);
+ let consumed = 0;
for (let index = runStart; index < runEnd; index += 1) {
- const point = left + (usableRight - left) * (index - runStart + 1) / (runEnd - runStart + 1);
+ const weight = weights[index - runStart]!;
+ const startSec = left + (usableRight - left) * consumed / totalWeight;
+ consumed += weight;
+ const endSec = left + (usableRight - left) * consumed / totalWeight;
values[index] = {
- startSec: point,
- endSec: point,
+ startSec,
+ endSec,
startQuality: "estimated",
endQuality: "estimated",
};
@@ -323,15 +415,27 @@ function assertMonotonic(tokens: readonly TimedSpeechToken[], segmentId: string)
}
}
-function mapDigest(map: Omit<CompleteSpeechTimeMap, "mapDigest">): string {
+function mapDigest(map: Omit<CompleteSemanticMap, "mapDigest">): CompleteSemanticMap["mapDigest"] {
return digestOf(map);
}
+function frameFor(basis: SpeechBasis, timeSec: number): number {
+ const { numerator, denominator } = basis.programSpace.frameRate;
+ return Math.round(timeSec * numerator / denominator);
+}
+
+function secondsFor(basis: SpeechBasis, frame: number): number {
+ const { numerator, denominator } = basis.programSpace.frameRate;
+ return frame * denominator / numerator;
+}
+
export function locateSpeechTiming(
- narrative: ParsedNarrative,
+ narrative: Narrative,
+ basis: SpeechBasis,
evidence: AlignedTranscriptEvidence,
-): CompleteSpeechTimeMap {
- validateEvidence(narrative, evidence);
+): CompleteSemanticMap {
+ validateBasis(narrative, basis);
+ validateEvidence(narrative, basis, evidence);
const evidenceBySegment = new Map(evidence.segments.map((segment) => [segment.sourceSegmentId, segment]));
const timedSegments: TimedSpeechSegment[] = [];
const timedTokens: TimedSpeechToken[] = [];
@@ -345,18 +449,32 @@ export function locateSpeechTiming(
const located: Array<MutableTiming | undefined> = Array(source.length).fill(undefined);
for (const group of segmentGroups) locatePairedGroup(group, source, aligned.words, chars, located);
const bounds = speechBounds(aligned);
- const complete = fillEstimated(located, bounds.start, bounds.end);
- const segmentTokens = source.map((token, index): TimedSpeechToken => ({
- tokenId: token.id,
- segmentId: segment.id,
- ...complete[index]!,
- }));
+ const complete = fillEstimated(located, source, bounds.start, bounds.end);
+ const segmentTokens = source.map((token, index): TimedSpeechToken => {
+ const timing = complete[index]!;
+ const startFrame = frameFor(basis, timing.startSec);
+ const endFrame = Math.max(startFrame, frameFor(basis, timing.endSec));
+ return {
+ tokenId: token.id,
+ segmentId: segment.id,
+ startSec: secondsFor(basis, startFrame),
+ endSec: secondsFor(basis, endFrame),
+ startFrame,
+ endFrame,
+ startQuality: timing.startQuality,
+ endQuality: timing.endQuality,
+ };
+ });
assertMonotonic(segmentTokens, segment.id);
timedTokens.push(...segmentTokens);
+ const startFrame = frameFor(basis, aligned.startSec);
+ const endFrame = Math.max(startFrame, frameFor(basis, aligned.endSec));
timedSegments.push({
segmentId: segment.id,
- startSec: aligned.startSec,
- endSec: aligned.endSec,
+ startSec: secondsFor(basis, startFrame),
+ endSec: secondsFor(basis, endFrame),
+ startFrame,
+ endFrame,
startQuality: "measured",
endQuality: "measured",
});
@@ -369,17 +487,23 @@ export function locateSpeechTiming(
if (anchor.kind === "segment-start" || anchor.kind === "segment-end") {
const segment = segmentsById.get(anchor.segmentId)!;
return anchor.kind === "segment-start"
- ? { identity: anchor.id, timeSec: segment.startSec, quality: segment.startQuality }
- : { identity: anchor.id, timeSec: segment.endSec, quality: segment.endQuality };
+ ? { identity: anchor.id, timeSec: segment.startSec, frame: segment.startFrame, quality: segment.startQuality }
+ : { identity: anchor.id, timeSec: segment.endSec, frame: segment.endFrame, quality: segment.endQuality };
}
const token = tokensById.get(anchor.tokenId!)!;
return anchor.kind === "token-start"
- ? { identity: anchor.id, timeSec: token.startSec, quality: token.startQuality }
- : { identity: anchor.id, timeSec: token.endSec, quality: token.endQuality };
+ ? { identity: anchor.id, timeSec: token.startSec, frame: token.startFrame, quality: token.startQuality }
+ : { identity: anchor.id, timeSec: token.endSec, frame: token.endFrame, quality: token.endQuality };
});
const payload = {
- contract: "svml.speech-time-map@0" as const,
+ contract: "svml.complete-semantic-map@1" as const,
semanticIndexDigest: narrative.semanticIndex.digest,
+ basisDigest: basis.basisDigest,
+ audioArtifactDigest: basis.audio.digest,
+ programSpaceDigest: basis.programSpace.digest,
+ evidenceDigest: evidence.evidenceDigest,
+ locatorDigest: speechLocatorDigest,
+ quantizationPolicy: "nearest-frame" as const,
durationSec: evidence.durationSec,
segments: timedSegments,
tokens: timedTokens,
diff --git a/packages/speech-align/src/manifest.ts b/packages/speech-align/src/manifest.ts
new file mode 100644
index 00000000..daf99431
--- /dev/null
+++ b/packages/speech-align/src/manifest.ts
@@ -0,0 +1,37 @@
+import {
+ contractTypes,
+ contractsManifestDigest,
+ contractsModuleRef,
+} from "@svml/contracts";
+import type { ModuleManifest, ProducerRef } from "@svml/protocol";
+
+import { speechLocatorDigest } from "./locate.js";
+
+export const speechAlignModuleRef = { name: "@svml/speech-align", version: "0.0.0-dev" } as const;
+export const speechAlignProducers = {
+ locate: { module: speechAlignModuleRef, name: "locate-speech" },
+} satisfies Record<string, ProducerRef>;
+
+export const speechAlignManifest: ModuleManifest = {
+ format: "svml.module@0",
+ name: speechAlignModuleRef.name,
+ version: speechAlignModuleRef.version,
+ dependencies: [{ module: contractsModuleRef, digest: contractsManifestDigest }],
+ types: [],
+ surfaces: [],
+ producers: [{
+ name: speechAlignProducers.locate.name,
+ inputs: [
+ { name: "narrative", type: contractTypes.narrative },
+ { name: "basis", type: contractTypes.speechBasis },
+ { name: "evidence", type: contractTypes.alignedTranscriptEvidence },
+ ],
+ outputs: [{ name: "map", type: contractTypes.completeSemanticMap }],
+ needs: [],
+ implementation: {
+ kind: "registered",
+ locator: "@svml/speech-align/locate",
+ digest: speechLocatorDigest,
+ },
+ }],
+};
diff --git a/packages/speech-align/src/types.ts b/packages/speech-align/src/types.ts
index d312b99a..bcf9b5fc 100644
--- a/packages/speech-align/src/types.ts
+++ b/packages/speech-align/src/types.ts
@@ -1,92 +1,15 @@
-export type TimingQuality = "measured" | "derived" | "estimated";
-
-export type SpeechWordEvidence = {
- readonly text: string;
- readonly startSec?: number;
- readonly endSec?: number;
- readonly score?: number;
-};
-
-export type SpeechCharacterEvidence = {
- readonly char: string;
- readonly wordIndex: number;
- readonly startSec?: number;
- readonly endSec?: number;
- readonly score?: number;
-};
-
-export type SpeechActivitySpan = {
- readonly startSec: number;
- readonly endSec: number;
-};
-
-/**
- * Provider-neutral evidence produced by a speech-recognition adapter. The
- * adapter keeps its raw provider result as an artifact and lowers only the
- * fields used by this deterministic locator into this value.
- */
-export type AlignedTranscriptSegment = {
- readonly sourceSegmentId: string;
- readonly startSec: number;
- readonly endSec: number;
- readonly words: readonly SpeechWordEvidence[];
- readonly chars: readonly SpeechCharacterEvidence[];
- readonly speechActivity?: readonly SpeechActivitySpan[];
-};
-
-export type AlignedTranscriptEvidence = {
- readonly contract: "svml.aligned-transcript-evidence@0";
- readonly durationSec: number;
- readonly segments: readonly AlignedTranscriptSegment[];
-};
-
-export type AlignmentRelation =
- | "exact"
- | "split"
- | "merge"
- | "replacement"
- | "source-omission"
- | "evidence-insertion";
-
-export type AlignmentGroup = {
- readonly sourceSegmentId: string;
- readonly sourceTokenIds: readonly string[];
- readonly evidenceWordStart: number;
- readonly evidenceWordEndExclusive: number;
- readonly relation: AlignmentRelation;
- readonly cost: number;
-};
-
-export type TimedSpeechToken = {
- readonly tokenId: string;
- readonly segmentId: string;
- readonly startSec: number;
- readonly endSec: number;
- readonly startQuality: TimingQuality;
- readonly endQuality: TimingQuality;
-};
-
-export type TimedSpeechSegment = {
- readonly segmentId: string;
- readonly startSec: number;
- readonly endSec: number;
- readonly startQuality: TimingQuality;
- readonly endQuality: TimingQuality;
-};
-
-export type SemanticTimePoint = {
- readonly identity: string;
- readonly timeSec: number;
- readonly quality: TimingQuality;
-};
-
-export type CompleteSpeechTimeMap = {
- readonly contract: "svml.speech-time-map@0";
- readonly semanticIndexDigest: string;
- readonly durationSec: number;
- readonly segments: readonly TimedSpeechSegment[];
- readonly tokens: readonly TimedSpeechToken[];
- readonly anchors: readonly SemanticTimePoint[];
- readonly groups: readonly AlignmentGroup[];
- readonly mapDigest: string;
-};
+export type {
+ AlignedTranscriptEvidence,
+ AlignedTranscriptSegment,
+ AlignmentGroup,
+ AlignmentRelation,
+ CompleteSemanticMap,
+ SemanticTimePoint,
+ SpeechActivitySpan,
+ SpeechBasis,
+ SpeechCharacterEvidence,
+ SpeechWordEvidence,
+ TimedSpeechSegment,
+ TimedSpeechToken,
+ TimingQuality,
+} from "@svml/contracts";
diff --git a/packages/speech-align/test/speech-align.test.ts b/packages/speech-align/test/speech-align.test.ts
index 6d4f21a6..30e29311 100644
--- a/packages/speech-align/test/speech-align.test.ts
+++ b/packages/speech-align/test/speech-align.test.ts
@@ -1,6 +1,13 @@
import assert from "node:assert/strict";
import test from "node:test";
+import {
+ sealAlignedTranscriptEvidence,
+ sealProgramSpace,
+ sealSpeechBasis,
+} from "@svml/contracts";
+import type { Narrative, SpeechBasis } from "@svml/contracts";
+import { digestOf } from "@svml/core";
import { parseScript } from "@svml/script";
import {
SpeechAlignmentError,
@@ -13,6 +20,7 @@ import type {
} from "@svml/speech-align";
function evidence(args: {
+ readonly basis: SpeechBasis;
readonly segmentId?: string;
readonly durationSec?: number;
readonly startSec?: number;
@@ -21,20 +29,74 @@ function evidence(args: {
readonly chars?: readonly SpeechCharacterEvidence[];
readonly vad?: readonly { readonly startSec: number; readonly endSec: number }[];
}): AlignedTranscriptEvidence {
- return {
- contract: "svml.aligned-transcript-evidence@0",
- durationSec: args.durationSec ?? 2,
+ return sealAlignedTranscriptEvidence({
+ contract: "svml.aligned-transcript-evidence@1",
+ basisDigest: args.basis.basisDigest,
+ audioArtifactDigest: args.basis.audio.digest,
+ programSpaceDigest: args.basis.programSpace.digest,
+ rawEvidenceArtifactDigest: digestOf("fixture:whisperx:raw"),
+ durationSec: args.basis.programSpace.durationSec,
segments: [
{
sourceSegmentId: args.segmentId ?? "line",
startSec: args.startSec ?? 0,
- endSec: args.endSec ?? 2,
+ endSec: args.endSec ?? args.basis.programSpace.durationSec,
words: args.words,
chars: args.chars ?? [],
...(args.vad === undefined ? {} : { speechActivity: args.vad }),
},
],
- };
+ });
+}
+
+function speechBasis(
+ narrative: Narrative,
+ durationSec = 2,
+ windows?: readonly { readonly startSec: number; readonly endSec: number }[],
+): SpeechBasis {
+ const programSpace = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec,
+ frameRate: { numerator: 1_000, denominator: 1 },
+ });
+ const audioDigest = digestOf(`fixture:audio:${narrative.semanticIndex.digest}:${durationSec}`);
+ const segments = narrative.segments.map((segment, index) => ({
+ segmentId: segment.id,
+ startSec: windows?.[index]?.startSec ?? durationSec * index / narrative.segments.length,
+ endSec: windows?.[index]?.endSec ?? durationSec * (index + 1) / narrative.segments.length,
+ sourceArtifactDigest: digestOf(`fixture:clip:${segment.id}`),
+ }));
+ return sealSpeechBasis({
+ contract: "svml.speech-basis@1",
+ programSpace,
+ audio: { digest: audioDigest, size: 1, mediaType: "audio/wav", durationSec },
+ visualTrack: {
+ clips: segments.map((segment) => ({
+ segmentId: segment.segmentId,
+ artifact: {
+ digest: segment.sourceArtifactDigest,
+ size: 1,
+ mediaType: "video/mp4",
+ durationSec: segment.endSec - segment.startSec,
+ },
+ startSec: segment.startSec,
+ endSec: segment.endSec,
+ })),
+ },
+ segments,
+ });
+}
+
+function locate(
+ narrative: Narrative,
+ args: Omit<Parameters<typeof evidence>[0], "basis">,
+) {
+ const durationSec = args.durationSec ?? 2;
+ const basis = speechBasis(narrative, durationSec, [{
+ startSec: args.startSec ?? 0,
+ endSec: args.endSec ?? durationSec,
+ }]);
+ return locateSpeechTiming(narrative, basis, evidence({ ...args, basis }));
}
function characters(
@@ -54,12 +116,12 @@ function characters(
test("exact transcript words cover every Script and Segment anchor", () => {
const narrative = parseScript("exact.svml", "<line>Hello world.</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
words: [
{ text: "Hello", startSec: 0.1, endSec: 0.4, score: 0.97 },
{ text: "world", startSec: 0.5, endSec: 0.9, score: 0.96 },
],
- }));
+ });
assert.equal(map.tokens.length, 2);
assert.equal(map.anchors.length, 2 * narrative.tokens.length + 2 * narrative.segments.length);
@@ -76,7 +138,7 @@ test("exact transcript words cover every Script and Segment anchor", () => {
test("M:1 uses evidence character times instead of dividing a merged word by length", () => {
const narrative = parseScript("merge.svml", "<line>can not</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
endSec: 1,
durationSec: 1,
words: [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }],
@@ -85,7 +147,7 @@ test("M:1 uses evidence character times instead of dividing a merged word by len
[0.1, 0.18, 0.27, 0.42, 0.51, 0.63],
[0.17, 0.26, 0.36, 0.5, 0.62, 0.78],
),
- }));
+ });
assert.equal(map.groups[0]?.relation, "merge");
assert.deepEqual(
@@ -99,12 +161,12 @@ test("M:1 uses evidence character times instead of dividing a merged word by len
test("1:N wraps all evidence words in one Script token", () => {
const narrative = parseScript("split.svml", "<line>website</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
words: [
{ text: "web", startSec: 0.2, endSec: 0.45, score: 0.9 },
{ text: "site", startSec: 0.5, endSec: 0.82, score: 0.91 },
],
- }));
+ });
assert.equal(map.groups[0]?.relation, "split");
assert.deepEqual(
@@ -115,7 +177,7 @@ test("1:N wraps all evidence words in one Script token", () => {
test("a recognized filler stays an insertion and does not absorb neighboring Script words", () => {
const narrative = parseScript("insertion.svml", "<line>I really like it.</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
words: [
{ text: "I", startSec: 0.1, endSec: 0.2, score: 0.98 },
{ text: "uh", startSec: 0.24, endSec: 0.34, score: 0.88 },
@@ -123,7 +185,7 @@ test("a recognized filler stays an insertion and does not absorb neighboring Scr
{ text: "like", startSec: 0.67, endSec: 0.82, score: 0.96 },
{ text: "it", startSec: 0.86, endSec: 0.96, score: 0.96 },
],
- }));
+ });
assert.deepEqual(map.groups.map((group) => group.relation), [
"exact",
@@ -143,15 +205,15 @@ test("a recognized filler stays an insertion and does not absorb neighboring Scr
);
});
-test("an omitted Script word gets a zero-width estimated point between measured neighbors", () => {
+test("an omitted Script word receives the complete unmeasured interval between neighbors", () => {
const narrative = parseScript("omission.svml", "<line>This is very good.</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
words: [
{ text: "This", startSec: 0.1, endSec: 0.25, score: 0.98 },
{ text: "is", startSec: 0.3, endSec: 0.4, score: 0.97 },
{ text: "good", startSec: 0.6, endSec: 0.82, score: 0.98 },
],
- }));
+ });
assert.deepEqual(map.groups.map((group) => group.relation), [
"exact",
@@ -161,30 +223,38 @@ test("an omitted Script word gets a zero-width estimated point between measured
]);
assert.deepEqual(
[map.tokens[2]?.startSec, map.tokens[2]?.endSec, map.tokens[2]?.startQuality],
- [0.5, 0.5, "estimated"],
+ [0.4, 0.6, "estimated"],
);
});
test("VAD bounds contain estimates when an entire Script Segment has no recognized words", () => {
const narrative = parseScript("vad.svml", "<line>One two.</line>");
- const map = locateSpeechTiming(narrative, evidence({
+ const map = locate(narrative, {
words: [],
vad: [{ startSec: 0.4, endSec: 1.2 }],
- }));
+ });
assert.deepEqual(
map.tokens.map((token) => [token.startSec, token.endSec, token.startQuality]),
[
- [0.4 + 0.8 / 3, 0.4 + 0.8 / 3, "estimated"],
- [0.4 + 1.6 / 3, 0.4 + 1.6 / 3, "estimated"],
+ [0.4, 0.8, "estimated"],
+ [0.8, 1.2, "estimated"],
],
);
});
test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {
const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");
- const map = locateSpeechTiming(narrative, {
- contract: "svml.aligned-transcript-evidence@0",
+ const basis = speechBasis(narrative, 2, [
+ { startSec: 0, endSec: 1 },
+ { startSec: 1, endSec: 2 },
+ ]);
+ const map = locateSpeechTiming(narrative, basis, sealAlignedTranscriptEvidence({
+ contract: "svml.aligned-transcript-evidence@1",
+ basisDigest: basis.basisDigest,
+ audioArtifactDigest: basis.audio.digest,
+ programSpaceDigest: basis.programSpace.digest,
+ rawEvidenceArtifactDigest: digestOf("fixture:segments:raw"),
durationSec: 2,
segments: [
{
@@ -202,7 +272,7 @@ test("multiple Script Segments stay independent even when evidence records arriv
chars: [],
},
],
- });
+ }));
assert.deepEqual(map.tokens.map((token) => [token.segmentId, token.startSec, token.endSec]), [
["one", 0.2, 0.55],
@@ -214,12 +284,62 @@ test("multiple Script Segments stay independent even when evidence records arriv
test("invalid overlapping evidence word windows fail instead of producing a reversed map", () => {
const narrative = parseScript("bad.svml", "<line>one two</line>");
assert.throws(
- () => locateSpeechTiming(narrative, evidence({
+ () => locate(narrative, {
words: [
{ text: "one", startSec: 0.1, endSec: 0.5 },
{ text: "two", startSec: 0.4, endSec: 0.8 },
],
- })),
+ }),
(error: unknown) => error instanceof SpeechAlignmentError && error.code === "SPEECH_WORD_ORDER",
);
});
+
+test("Evidence from another same-duration audio Basis is rejected before alignment", () => {
+ const narrative = parseScript("affinity.svml", "<line>Hello world.</line>");
+ const firstBasis = speechBasis(narrative, 2);
+ const firstEvidence = evidence({
+ basis: firstBasis,
+ words: [
+ { text: "Hello", startSec: 0.1, endSec: 0.4 },
+ { text: "world", startSec: 0.5, endSec: 0.9 },
+ ],
+ });
+ const secondProgram = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec: 2,
+ frameRate: { numerator: 1_000, denominator: 1 },
+ });
+ const { basisDigest: _firstDigest, ...firstContent } = firstBasis;
+ const secondBasis = sealSpeechBasis({
+ ...firstContent,
+ programSpace: secondProgram,
+ audio: { ...firstBasis.audio, digest: digestOf("fixture:another-same-duration-audio") },
+ });
+ assert.throws(
+ () => locateSpeechTiming(narrative, secondBasis, firstEvidence),
+ (error: unknown) => error instanceof SpeechAlignmentError && error.code === "SPEECH_EVIDENCE_BASIS",
+ );
+});
+
+test("the final map is quantized once into the selected ProgramSpace", () => {
+ const narrative = parseScript("frames.svml", "<line>Hello.</line>");
+ const original = speechBasis(narrative, 1);
+ const programSpace = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec: 1,
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ const { basisDigest: _originalDigest, ...originalContent } = original;
+ const basis = sealSpeechBasis({ ...originalContent, programSpace });
+ const map = locateSpeechTiming(narrative, basis, evidence({
+ basis,
+ durationSec: 1,
+ endSec: 1,
+ words: [{ text: "Hello", startSec: 0.111, endSec: 0.289 }],
+ }));
+ assert.deepEqual(
+ [map.tokens[0]?.startFrame, map.tokens[0]?.endFrame, map.tokens[0]?.startSec, map.tokens[0]?.endSec],
+ [3, 9, 0.1, 0.3],
+ );
+ assert.equal(map.anchors.every((anchor) => anchor.timeSec === anchor.frame / 30), true);
+});
diff --git a/packages/text/package.json b/packages/text/package.json
index 5df0f6ae..09d5ae4c 100644
--- a/packages/text/package.json
+++ b/packages/text/package.json
@@ -11,6 +11,7 @@
"@svml/protocol": "workspace:*"
},
"devDependencies": {
+ "@svml/contracts": "workspace:*",
"@svml/script": "workspace:*"
}
}
diff --git a/packages/text/test/text.test.ts b/packages/text/test/text.test.ts
index abf4526e..3f1b8ef4 100644
--- a/packages/text/test/text.test.ts
+++ b/packages/text/test/text.test.ts
@@ -1,6 +1,7 @@
import assert from "node:assert/strict";
import test from "node:test";
+import { contractsManifest } from "@svml/contracts";
import { createResolvedClosure, digestOf, isDigest, link } from "@svml/core";
import type { ModuleManifest } from "@svml/protocol";
import {
@@ -17,7 +18,7 @@ import {
} from "@svml/text";
function scriptContext() {
- const closure = createResolvedClosure([scriptManifest]);
+ const closure = createResolvedClosure([contractsManifest, scriptManifest]);
const registry = new TextSurfaceRegistry();
registry.registerRaw(
scriptModuleRef,
@@ -99,7 +100,7 @@ test("without the import, Text has no hard-coded knowledge of Script", () => {
});
test("a raw Surface cannot consume the Text document close", () => {
- const closure = createResolvedClosure([scriptManifest]);
+ const closure = createResolvedClosure([contractsManifest, scriptManifest]);
const registry = new TextSurfaceRegistry();
registry.registerRaw(
scriptModuleRef,
diff --git a/packages/whisperx/README.md b/packages/whisperx/README.md
new file mode 100644
index 00000000..d33e2d33
--- /dev/null
+++ b/packages/whisperx/README.md
@@ -0,0 +1,9 @@
+# @svml/whisperx
+
+Explicit WhisperX model-family capability for the official speech program. Importing this package
+selects WhisperX; Runtime registration only binds the resulting `WhisperXAlignmentEvidence` Need to
+a concrete WhisperX execution endpoint such as a trusted local worker or Hypit.
+
+The package contains no credentials, Python environment or queue. Providers translate the typed
+request constraints and return the canonical WhisperX result contract. A deterministic Producer
+then lowers that result to provider-neutral `AlignedTranscriptEvidence`.
diff --git a/packages/whisperx/package.json b/packages/whisperx/package.json
new file mode 100644
index 00000000..c3e8e952
--- /dev/null
+++ b/packages/whisperx/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "@svml/whisperx",
+ "version": "0.0.0-dev",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "dependencies": {
+ "@svml/contracts": "workspace:*",
+ "@svml/core": "workspace:*",
+ "@svml/protocol": "workspace:*"
+ }
+}
diff --git a/packages/whisperx/src/evidence.ts b/packages/whisperx/src/evidence.ts
new file mode 100644
index 00000000..9f8d0bf1
--- /dev/null
+++ b/packages/whisperx/src/evidence.ts
@@ -0,0 +1,49 @@
+import {
+ assertSpeechBasisIdentity,
+ sealAlignedTranscriptEvidence,
+} from "@svml/contracts";
+import type { AlignedTranscriptEvidence, SpeechBasis } from "@svml/contracts";
+import { digestOf } from "@svml/core";
+
+import type {
+ WhisperXAlignmentEvidence,
+ WhisperXAlignmentRequest,
+ WhisperXEvidenceContent,
+} from "./types.js";
+
+export function whisperXRequestForBasis(
+ basis: SpeechBasis,
+ options: { readonly language?: string } = {},
+): WhisperXAlignmentRequest {
+ assertSpeechBasisIdentity(basis);
+ return {
+ contract: "svml.whisperx-alignment-request@1",
+ basisDigest: basis.basisDigest,
+ programSpaceDigest: basis.programSpace.digest,
+ audio: basis.audio,
+ segments: basis.segments,
+ ...(options.language === undefined ? {} : { language: options.language }),
+ wordAlignment: true,
+ characterAlignment: "when-available",
+ };
+}
+
+export function sealWhisperXAlignmentEvidence(
+ content: WhisperXEvidenceContent,
+): WhisperXAlignmentEvidence {
+ return { ...content, alignmentDigest: digestOf(content) };
+}
+
+export function normalizeWhisperXAlignment(
+ evidence: WhisperXAlignmentEvidence,
+): AlignedTranscriptEvidence {
+ const { alignmentDigest, ...content } = evidence;
+ if (alignmentDigest !== digestOf(content)) {
+ throw new Error("WhisperX alignment digest does not match its canonical contents.");
+ }
+ const { alignmentDigest: _alignmentDigest, engine: _engine, contract: _contract, ...shared } = evidence;
+ return sealAlignedTranscriptEvidence({
+ contract: "svml.aligned-transcript-evidence@1",
+ ...shared,
+ });
+}
diff --git a/packages/whisperx/src/index.ts b/packages/whisperx/src/index.ts
new file mode 100644
index 00000000..7a516254
--- /dev/null
+++ b/packages/whisperx/src/index.ts
@@ -0,0 +1,14 @@
+export {
+ normalizeWhisperXAlignment,
+ sealWhisperXAlignmentEvidence,
+ whisperXRequestForBasis,
+} from "./evidence.js";
+export {
+ whisperXAlignmentEvidenceSchema,
+ whisperXImplementationDigests,
+ whisperXManifest,
+ whisperXModuleRef,
+ whisperXProducers,
+ whisperXTypes,
+} from "./manifest.js";
+export type * from "./types.js";
diff --git a/packages/whisperx/src/manifest.ts b/packages/whisperx/src/manifest.ts
new file mode 100644
index 00000000..f468ffb6
--- /dev/null
+++ b/packages/whisperx/src/manifest.ts
@@ -0,0 +1,66 @@
+import {
+ alignedTranscriptEvidenceFields,
+ contractTypes,
+ contractsManifestDigest,
+ contractsModuleRef,
+} from "@svml/contracts";
+import { digestOf } from "@svml/core";
+import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@svml/protocol";
+
+export const whisperXModuleRef = { name: "@svml/whisperx", version: "0.0.0-dev" } as const;
+export const whisperXTypes = {
+ alignmentEvidence: { module: whisperXModuleRef, name: "WhisperXAlignmentEvidence" },
+} satisfies Record<string, TypeRef>;
+export const whisperXProducers = {
+ request: { module: whisperXModuleRef, name: "request-whisperx-alignment" },
+ normalize: { module: whisperXModuleRef, name: "normalize-whisperx-alignment" },
+} satisfies Record<string, ProducerRef>;
+export const whisperXImplementationDigests = {
+ request: digestOf("@svml/whisperx/request@1"),
+ normalize: digestOf("@svml/whisperx/normalize@1"),
+};
+
+const { contract: _commonContract, evidenceDigest: _commonDigest, ...sharedEvidenceFields } =
+ alignedTranscriptEvidenceFields;
+export const whisperXAlignmentEvidenceSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ contract: { schema: { kind: "literal", value: "svml.whisperx-alignment-evidence@1" } },
+ engine: { schema: { kind: "literal", value: "whisperx" } },
+ ...sharedEvidenceFields,
+ alignmentDigest: { schema: { kind: "string", minLength: 71, maxLength: 71 } },
+ },
+};
+
+export const whisperXManifest: ModuleManifest = {
+ format: "svml.module@0",
+ name: whisperXModuleRef.name,
+ version: whisperXModuleRef.version,
+ dependencies: [{ module: contractsModuleRef, digest: contractsManifestDigest }],
+ types: [{ name: whisperXTypes.alignmentEvidence.name, schema: whisperXAlignmentEvidenceSchema }],
+ surfaces: [],
+ producers: [
+ {
+ name: whisperXProducers.request.name,
+ inputs: [{ name: "basis", type: contractTypes.speechBasis }],
+ outputs: [],
+ needs: [{ name: "alignment", wants: whisperXTypes.alignmentEvidence }],
+ implementation: {
+ kind: "registered",
+ locator: "@svml/whisperx/request",
+ digest: whisperXImplementationDigests.request,
+ },
+ },
+ {
+ name: whisperXProducers.normalize.name,
+ inputs: [{ name: "whisperx", type: whisperXTypes.alignmentEvidence }],
+ outputs: [{ name: "evidence", type: contractTypes.alignedTranscriptEvidence }],
+ needs: [],
+ implementation: {
+ kind: "registered",
+ locator: "@svml/whisperx/normalize",
+ digest: whisperXImplementationDigests.normalize,
+ },
+ },
+ ],
+};
diff --git a/packages/whisperx/src/types.ts b/packages/whisperx/src/types.ts
new file mode 100644
index 00000000..9e5aedf1
--- /dev/null
+++ b/packages/whisperx/src/types.ts
@@ -0,0 +1,33 @@
+import type {
+ AlignedTranscriptEvidence,
+ AlignedTranscriptSegment,
+ MediaArtifactRef,
+ SpeechBasisSegment,
+} from "@svml/contracts";
+import type { Digest } from "@svml/protocol";
+
+export type WhisperXAlignmentRequest = {
+ readonly contract: "svml.whisperx-alignment-request@1";
+ readonly basisDigest: Digest;
+ readonly programSpaceDigest: Digest;
+ readonly audio: MediaArtifactRef;
+ readonly segments: readonly SpeechBasisSegment[];
+ readonly language?: string;
+ readonly wordAlignment: true;
+ readonly characterAlignment: "when-available";
+};
+
+export type WhisperXAlignmentEvidence = {
+ readonly contract: "svml.whisperx-alignment-evidence@1";
+ readonly engine: "whisperx";
+ readonly basisDigest: Digest;
+ readonly audioArtifactDigest: Digest;
+ readonly programSpaceDigest: Digest;
+ readonly rawEvidenceArtifactDigest: Digest;
+ readonly durationSec: number;
+ readonly segments: readonly AlignedTranscriptSegment[];
+ readonly alignmentDigest: Digest;
+};
+
+export type WhisperXEvidenceContent = Omit<WhisperXAlignmentEvidence, "alignmentDigest">;
+export type ProviderNeutralEvidenceContent = Omit<AlignedTranscriptEvidence, "evidenceDigest">;
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.ts
new file mode 100644
index 00000000..55ae38df
--- /dev/null
+++ b/packages/whisperx/test/whisperx.test.ts
@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { sealProgramSpace, sealSpeechBasis } from "@svml/contracts";
+import { digestOf } from "@svml/core";
+import {
+ normalizeWhisperXAlignment,
+ sealWhisperXAlignmentEvidence,
+ whisperXRequestForBasis,
+} from "@svml/whisperx";
+
+function basis() {
+ const programSpace = sealProgramSpace({
+ contract: "svml.program-space@0",
+ durationSec: 1,
+ frameRate: { numerator: 30, denominator: 1 },
+ });
+ const visual = digestOf("whisperx-test:visual");
+ return sealSpeechBasis({
+ contract: "svml.speech-basis@1",
+ programSpace,
+ audio: { digest: digestOf("whisperx-test:audio"), size: 1, mediaType: "audio/wav", durationSec: 1 },
+ visualTrack: { clips: [{
+ segmentId: "line",
+ artifact: { digest: visual, size: 1, mediaType: "video/mp4", durationSec: 1 },
+ startSec: 0,
+ endSec: 1,
+ }] },
+ segments: [{ segmentId: "line", startSec: 0, endSec: 1, sourceArtifactDigest: visual }],
+ });
+}
+
+test("WhisperX request construction rejects a self-inconsistent SpeechBasis before fulfillment", () => {
+ const valid = basis();
+ assert.equal(whisperXRequestForBasis(valid).basisDigest, valid.basisDigest);
+ const tampered = { ...valid, audio: { ...valid.audio, digest: digestOf("another-audio") } };
+ assert.throws(() => whisperXRequestForBasis(tampered), /SpeechBasis digest/u);
+});
+
+test("WhisperX normalization verifies its model-specific result before lowering to common Evidence", () => {
+ const source = basis();
+ const measured = sealWhisperXAlignmentEvidence({
+ contract: "svml.whisperx-alignment-evidence@1",
+ engine: "whisperx",
+ basisDigest: source.basisDigest,
+ audioArtifactDigest: source.audio.digest,
+ programSpaceDigest: source.programSpace.digest,
+ rawEvidenceArtifactDigest: digestOf("whisperx-test:raw"),
+ durationSec: 1,
+ segments: [{
+ sourceSegmentId: "line",
+ startSec: 0,
+ endSec: 1,
+ words: [{ text: "hello", startSec: 0.1, endSec: 0.4 }],
+ chars: [],
+ }],
+ });
+ const normalized = normalizeWhisperXAlignment(measured);
+ assert.equal(normalized.contract, "svml.aligned-transcript-evidence@1");
+ assert.equal(normalized.basisDigest, source.basisDigest);
+ assert.throws(
+ () => normalizeWhisperXAlignment({
+ ...measured,
+ segments: [{ ...measured.segments[0]!, words: [{ text: "tampered", startSec: 0.1, endSec: 0.4 }] }],
+ }),
+ /alignment digest/u,
+ );
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f0e1de69..71313ab6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -24,9 +24,16 @@ importers:
packages/caption:
dependencies:
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
'@svml/core':
specifier: workspace:*
version: link:../core
+ '@svml/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ devDependencies:
'@svml/script':
specifier: workspace:*
version: link:../script
@@ -34,6 +41,15 @@ importers:
specifier: workspace:*
version: link:../speech-align
+ packages/contracts:
+ dependencies:
+ '@svml/core':
+ specifier: workspace:*
+ version: link:../core
+ '@svml/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+
packages/core:
dependencies:
'@svml/protocol':
@@ -48,11 +64,30 @@ importers:
'@svml/protocol':
specifier: workspace:*
version: link:../protocol
+ devDependencies:
+ '@svml/caption':
+ specifier: workspace:*
+ version: link:../caption
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
+ '@svml/script':
+ specifier: workspace:*
+ version: link:../script
+ '@svml/speech-align':
+ specifier: workspace:*
+ version: link:../speech-align
+ '@svml/whisperx':
+ specifier: workspace:*
+ version: link:../whisperx
packages/protocol: {}
packages/script:
dependencies:
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
'@svml/core':
specifier: workspace:*
version: link:../core
@@ -62,9 +97,16 @@ importers:
packages/speech-align:
dependencies:
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
'@svml/core':
specifier: workspace:*
version: link:../core
+ '@svml/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ devDependencies:
'@svml/script':
specifier: workspace:*
version: link:../script
@@ -78,10 +120,25 @@ importers:
specifier: workspace:*
version: link:../protocol
devDependencies:
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
'@svml/script':
specifier: workspace:*
version: link:../script
+ packages/whisperx:
+ dependencies:
+ '@svml/contracts':
+ specifier: workspace:*
+ version: link:../contracts
+ '@svml/core':
+ specifier: workspace:*
+ version: link:../core
+ '@svml/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+
packages:
'@emnapi/[email protected]':
diff --git a/tsconfig.v2.json b/tsconfig.v2.json
index 7c9cc4f4..84967af9 100644
--- a/tsconfig.v2.json
+++ b/tsconfig.v2.json
@@ -7,10 +7,12 @@
"paths": {
"@svml/protocol": ["packages/protocol/src/index.ts"],
"@svml/core": ["packages/core/src/index.ts"],
+ "@svml/contracts": ["packages/contracts/src/index.ts"],
"@svml/driver-node": ["packages/driver-node/src/index.ts"],
"@svml/script": ["packages/script/src/index.ts"],
"@svml/text": ["packages/text/src/index.ts"],
"@svml/speech-align": ["packages/speech-align/src/index.ts"],
+ "@svml/whisperx": ["packages/whisperx/src/index.ts"],
"@svml/caption": ["packages/caption/src/index.ts"]
}
},