Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: introduce per-segment semantic takes

kashorind91482f1125f
diff --git a/packages/caption/test/caption.test.ts b/packages/caption/test/caption.test.ts
index 379bd2b1..3997ce7e 100644
--- a/packages/caption/test/caption.test.ts
+++ b/packages/caption/test/caption.test.ts
@@ -17,9 +17,8 @@ import type { CaptionDisplayWordSubset, Narrative } from "@hypit/narrative";
import { canonicalize } from "@hypit/protocol";
import type { StoredValue, TypeRef } from "@hypit/protocol";
import { sealProgramSpace } from "@hypit/program-space";
-import { sealSpeechBasis } from "@hypit/speech";
-import type { SpeechAudioBasis } from "@hypit/speech";
-import { locateSpeechTiming } from "@hypit/speech-alignment";
+import { locateAlignedSegmentTiming } from "../../speech-alignment/src/locate.js";
+import type { AlignmentBasis } from "../../speech-alignment/src/locate.js";
import { sealAlignedTranscriptEvidence } from "@hypit/speech-evidence";
import {
captionCorrespondence,
@@ -40,22 +39,14 @@ type TranscriptFixture = {
};
function locate(narrative: Narrative, durationSec: number, segments: readonly TranscriptFixture[]) {
+ if (narrative.segments.length !== 1 || segments.length !== 1) {
+ throw new Error("Caption timing fixture requires one Segment.");
+ }
const space = sealProgramSpace({
durationSec,
frameRate: { numerator: 30, denominator: 1 },
});
const audio = { kind: "blob" as const, digest: fixtureDigest("caption:test-audio"), size: 1, mediaType: "audio/wav" };
- const basisSegments = narrative.segments.map((segment, index) => ({
- segmentId: segment.id,
- startFrame: Math.round(durationSec * 30 * index / narrative.segments.length),
- endFrameExclusive: Math.round(durationSec * 30 * (index + 1) / narrative.segments.length),
- }));
- const basis = sealSpeechBasis({
- programSpace: space,
- audio,
- visualTrack: { clips: [] },
- segments: basisSegments,
- });
const evidence = sealAlignedTranscriptEvidence({
passages: segments.map((segment) => ({
words: segment.words.map((word) => ({
@@ -69,12 +60,16 @@ function locate(narrative: Narrative, durationSec: number, segments: readonly Tr
chars: [],
})),
});
- const audioBasis: SpeechAudioBasis = {
- programSpace: basis.programSpace,
- audio: basis.audio,
- segments: basis.segments,
+ const audioBasis: AlignmentBasis = {
+ programSpace: space,
+ audio,
+ segments: [{
+ segmentId: narrative.segments[0]!.id,
+ startFrame: 0,
+ endFrameExclusive: Math.round(durationSec * 30),
+ }],
};
- return locateSpeechTiming(narrative, audioBasis, evidence);
+ return locateAlignedSegmentTiming(narrative, audioBasis, evidence);
}
function style(id: string, fields: readonly CaptionFieldDeclaration[] = []): CaptionStyleIntent {
diff --git a/packages/elaborator/test/fragment.test.ts b/packages/elaborator/test/fragment.test.ts
index 6a43b145..7187fe1c 100644
--- a/packages/elaborator/test/fragment.test.ts
+++ b/packages/elaborator/test/fragment.test.ts
@@ -100,12 +100,6 @@ function speechFragment(): GraphFragment {
inputs: { request: input("request"), style: input("style") },
result: { kind: "output", name: "take" },
},
- {
- id: "audio",
- producer: speechBasisProducers.projectAudio,
- inputs: { basis: operation("generate") },
- result: { kind: "output", name: "audio" },
- },
{
id: "visual",
producer: speechBasisProducers.projectVisual,
@@ -119,11 +113,6 @@ function speechFragment(): GraphFragment {
type: speechTypes.basis,
root: operation("generate"),
},
- {
- name: "audio",
- type: speechTypes.audioBasis,
- root: operation("audio"),
- },
{
name: "visual",
type: compositionTypes.visualTrack,
diff --git a/packages/markup/src/element.ts b/packages/markup/src/element.ts
new file mode 100644
index 00000000..4c32ed40
--- /dev/null
+++ b/packages/markup/src/element.ts
@@ -0,0 +1,39 @@
+import type { StructuredElement } from "./types.js";
+
+export function assertAttributes(
+ element: StructuredElement,
+ allowed: readonly string[],
+ required: readonly string[] = [],
+): void {
+ const unknown = Object.keys(element.attributes).filter((name) => !allowed.includes(name));
+ if (unknown.length > 0) throw new Error(`${element.name} does not accept ${unknown[0]}.`);
+ const missing = required.filter((name) => element.attributes[name] === undefined);
+ if (missing.length > 0) throw new Error(`${element.name} requires ${missing.join(", ")}.`);
+}
+
+export function assertExactAttributes(element: StructuredElement, names: readonly string[]): void {
+ assertAttributes(element, names, names);
+}
+
+export function assertEmptyElement(element: StructuredElement): void {
+ if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
+ throw new Error(`${element.name} must be empty.`);
+ }
+}
+
+export function localName(name: string): string {
+ return name.slice(name.lastIndexOf(":") + 1);
+}
+
+export function textAttribute(element: StructuredElement, name: string, fallback?: string): string {
+ const value = element.attributes[name];
+ if (value === undefined && fallback !== undefined) return fallback;
+ if (typeof value !== "string" || value.trim().length === 0) {
+ throw new Error(`${element.name}.${name} must be text.`);
+ }
+ return value.trim();
+}
+
+export function optionalTextAttribute(element: StructuredElement, name: string): string | undefined {
+ return element.attributes[name] === undefined ? undefined : textAttribute(element, name);
+}
diff --git a/packages/markup/src/index.ts b/packages/markup/src/index.ts
index e1c1eaf1..4b6ccbcc 100644
--- a/packages/markup/src/index.ts
+++ b/packages/markup/src/index.ts
@@ -1,4 +1,12 @@
export { MarkupFrontendError } from "./error.js";
+export {
+ assertAttributes,
+ assertEmptyElement,
+ assertExactAttributes,
+ localName,
+ optionalTextAttribute,
+ textAttribute,
+} from "./element.js";
export { decodeMarkup, createMarkupAuthorFrontend, markupAuthorFrontendId } from "./frontend.js";
export { MarkupSurfaceRegistry } from "./registry.js";
export {
diff --git a/packages/media-pipeline/README.md b/packages/media-pipeline/README.md
index 0c008bd2..6b65bcde 100644
--- a/packages/media-pipeline/README.md
+++ b/packages/media-pipeline/README.md
@@ -17,9 +17,8 @@ BlobArtifact
and otherwise fails closed. An explicit stream index is available when the author module genuinely
knows which alternate stream it wants.
-Selecting embedded audio is a media fact only. This package never produces `SpeechBasis` or
-`SpeechAudioBasis`; a speech component must separately bind normalized audio to a Narrative and its
-Segment identities.
+Selecting embedded audio is a media fact only. This package never assigns Script meaning. A later
+speech alignment component binds one normalized Take to one authored Segment.
The current normalization profile uses one source presentation origin, a requested rational video frame
rate, and a 48 kHz stereo PCM render stem. It preserves input level: loudness/mastering remains a
@@ -28,18 +27,28 @@ optional visual Artifact plus intrinsic extent, and optional audio Artifact. Str
authority choice and the trim/pad ledger remain in Selection and execution instead of travelling
through every consumer.
-This package is the shared execution vocabulary, not mandatory authoring boilerplate. A consuming
-Surface that already owns the needed policy may expand the same finite graph itself. For example,
-`speech:Take video={take.video}` selects default A/V at the Spine's explicit frame rate, while
-`media-track:Item video={shot.video}` selects visual-only media at the connected ProgramSpace rate.
-Their lower-level `media=` inputs remain available for an explicitly prepared `SynchronizedMedia`.
-Core and Providers see the same Operations in both forms.
+This package is the shared execution vocabulary. Speech authoring makes the boundary explicit:
+`<pipeline:Normalize>` produces one `SynchronizedMedia`, then a speech provider and deterministic
+alignment produce one `SemanticTake`. Media Track consumes that prepared value; it never owns an
+implicit visual-only normalization policy.
+
+The frame clock is a separate program fact and can be shared by every Normalize and Speech Spine:
+
+```svml
+<program:Clock id="clock" frame-rate="30"/>
+<pipeline:Normalize id="shot-media" source={shot.video}
+ recipe={recipes.media.aroll} clock={clock}/>
+```
+
+An SVS recipe may centralize the stream policy (`video`, `audio` and `span-authority`); it does not
+hide the Normalize graph node or turn multiple sources into one opaque batch operation.
Three ordinary author operations reuse that same inspection/execution boundary:
```xml
-<media:Transform id="prepared" source={shot.video}
- video="primary-moving" audio="default" span-authority="video" frame-rate="30">
+<pipeline:Normalize id="shot-media" source={shot.video}
+ video="primary-moving" audio="default" span-authority="video" clock={clock}/>
+<media:Transform id="prepared" source={shot-media.media}>
<media:Trim tail="0.25s"/>
<media:Retime rate="1.05" pitch="preserve"/>
</media:Transform>
diff --git a/packages/media-pipeline/package.json b/packages/media-pipeline/package.json
index acb6eb94..7f1ae587 100644
--- a/packages/media-pipeline/package.json
+++ b/packages/media-pipeline/package.json
@@ -20,6 +20,7 @@
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
"@hypit/speech": "workspace:*",
+ "@hypit/svs": "workspace:*",
"@hypit/markup": "workspace:*"
}
}
diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.ts
index db04d9e8..1d2d1083 100644
--- a/packages/media-pipeline/src/component.ts
+++ b/packages/media-pipeline/src/component.ts
@@ -1,11 +1,9 @@
import { mediaComponent } from "@hypit/media";
import type { ComponentPackage } from "@hypit/component-kit";
-import { verifyMediaInspection, verifyMediaStreamSelection, verifyMuxedMedia, verifyRenderedVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
-import type { MediaInspection, MediaStreamSelection, MuxedMedia, RenderedVisual, TimelineAudio } from "@hypit/media";
-import { assertProgramSpaceIdentity, programSpaceSampleFrames } from "@hypit/program-space";
+import { synchronizedMediaSampleFrames, verifyMediaInspection, verifyMediaStreamSelection, verifyMuxedMedia, verifyRenderedVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
+import type { MediaInspection, MediaStreamSelection, MuxedMedia, RenderedVisual, SynchronizedMedia, TimelineAudio } from "@hypit/media";
import type { ProgramSpace } from "@hypit/program-space";
-import { assertSpeechAudioBasisIdentity, speechEvidenceSampleBoundary } from "@hypit/speech";
-import type { SpeechAudioBasis } from "@hypit/speech";
+import { speechEvidenceSampleBoundary } from "@hypit/speech";
import type { Composition } from "@hypit/composition";
import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
import { canonicalize } from "@hypit/protocol";
@@ -16,7 +14,6 @@ import {
} from "./audio-plan.js";
import { mediaPipelineProducers, mediaPipelineTypes } from "./manifest.js";
import {
- sealMediaSelectionRequest,
selectMediaStreams,
verifyMediaSelectionRequest,
} from "./selection.js";
@@ -51,21 +48,6 @@ function blob(value: StoredValue, subject: string): BlobRef {
return value;
}
-function programSpace(value: StoredValue, subject: string): ProgramSpace {
- const space = inline(value, subject) as unknown as ProgramSpace;
- assertProgramSpaceIdentity(space);
- return space;
-}
-
-function requestForProgram(space: ProgramSpace, audio: "default" | "none"): MediaSelectionRequest {
- return sealMediaSelectionRequest({
- video: { mode: "primary-moving" },
- audio: { mode: audio },
- spanAuthority: "video",
- frameRate: { ...space.frameRate },
- });
-}
-
export const mediaPipelineComponent = {
validators: [
{
@@ -100,36 +82,6 @@ export const mediaPipelineComponent = {
},
],
producers: [
- {
- producer: mediaPipelineProducers.bindVisualRequest,
- handler: ({ inputs }) => ({
- outputs: {
- request: {
- kind: "inline",
- value: canonicalize(requestForProgram(
- programSpace(inputs.space!.value, "Visual media ProgramSpace"),
- "none",
- )),
- },
- },
- needs: {},
- }),
- },
- {
- producer: mediaPipelineProducers.bindAvRequest,
- handler: ({ inputs }) => ({
- outputs: {
- request: {
- kind: "inline",
- value: canonicalize(requestForProgram(
- programSpace(inputs.space!.value, "A/V media ProgramSpace"),
- "default",
- )),
- },
- },
- needs: {},
- }),
- },
{
producer: mediaPipelineProducers.inspect,
handler: ({ inputs }) => {
@@ -230,21 +182,17 @@ export const mediaPipelineComponent = {
{
producer: mediaPipelineProducers.projectSpeechEvidenceAudio,
handler: ({ inputs }) => {
- const audio = inline(inputs.audio!.value, "SpeechAudioBasis") as unknown as SpeechAudioBasis;
- assertSpeechAudioBasisIdentity(audio);
- const sourceSampleFrames = programSpaceSampleFrames(audio.programSpace, 48_000);
+ const media = inline(inputs.media!.value, "SynchronizedMedia") as unknown as SynchronizedMedia;
+ verifySynchronizedMedia(media);
+ if (media.audio === undefined) throw new Error("Speech evidence requires normalized Take audio");
+ const sourceSampleFrames = synchronizedMediaSampleFrames(media);
const evidenceSampleFrames = speechEvidenceSampleBoundary(sourceSampleFrames);
if (!Number.isSafeInteger(sourceSampleFrames) || sourceSampleFrames < 1
|| !Number.isSafeInteger(evidenceSampleFrames) || evidenceSampleFrames < 1) {
throw new Error("Speech evidence audio sample domain is invalid");
}
const need: ProjectSpeechEvidenceAudioNeed = {
- source: {
- kind: "blob",
- digest: audio.audio.digest,
- size: audio.audio.size,
- mediaType: audio.audio.mediaType,
- },
+ source: media.audio.artifact,
sourceSampleFrames,
evidenceSampleFrames,
};
diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.ts
index 2132ed26..2acd5735 100644
--- a/packages/media-pipeline/src/fragment.ts
+++ b/packages/media-pipeline/src/fragment.ts
@@ -49,38 +49,14 @@ export const synchronizedMediaFragment = sealGraphFragment({
export const transformMediaFragment = sealGraphFragment({
inputs: [
- { name: "source", type: artifactTypes.blob },
- { name: "selection", type: mediaPipelineTypes.selectionRequest },
+ { name: "media", type: mediaTypes.synchronized },
{ name: "program", type: mediaPipelineTypes.transformProgram },
],
operations: [
- {
- id: "inspect",
- producer: mediaPipelineProducers.inspect,
- inputs: { source: input("source") },
- result: { kind: "need", name: "inspection" },
- },
- {
- id: "select",
- producer: mediaPipelineProducers.select,
- inputs: { inspection: operation("inspect"), request: input("selection") },
- result: { kind: "output", name: "selection" },
- },
- {
- id: "normalize",
- producer: mediaPipelineProducers.normalize,
- inputs: {
- source: input("source"),
- inspection: operation("inspect"),
- selection: operation("select"),
- request: input("selection"),
- },
- result: { kind: "need", name: "media" },
- },
{
id: "transform",
producer: mediaPipelineProducers.transform,
- inputs: { media: operation("normalize"), program: input("program") },
+ inputs: { media: input("media"), program: input("program") },
result: { kind: "need", name: "video" },
},
],
diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.ts
index a8e5ea87..18bddc3e 100644
--- a/packages/media-pipeline/src/manifest.ts
+++ b/packages/media-pipeline/src/manifest.ts
@@ -1,6 +1,7 @@
import { artifactDependency } from "@hypit/artifact";
import { mediaDependency, mediaTypes } from "@hypit/media";
import { programSpaceDependency, programSpaceTypes } from "@hypit/program-space";
+import { svsModuleRef, svsRecipeType } from "@hypit/svs";
import { speechDependency, speechTypes } from "@hypit/speech";
import { compositionDependency, compositionTypes } from "@hypit/composition";
import { artifactTypes } from "@hypit/artifact";
@@ -31,8 +32,6 @@ export const mediaPipelineCapabilities = {
mux: { module: mediaPipelineModuleRef, name: "mux-program-media" },
} satisfies Record<string, CapabilityRef>;
export const mediaPipelineProducers = {
- bindVisualRequest: { module: mediaPipelineModuleRef, name: "bind-visual-media-request-to-program" },
- bindAvRequest: { module: mediaPipelineModuleRef, name: "bind-av-media-request-to-program" },
inspect: { module: mediaPipelineModuleRef, name: "request-media-inspection" },
select: { module: mediaPipelineModuleRef, name: "select-media-streams" },
normalize: { module: mediaPipelineModuleRef, name: "request-media-normalization" },
@@ -211,24 +210,28 @@ export const mediaPipelineMarkupSurfaces = [
{ name: "source", kind: "reference", required: true,
accepts: [artifactTypes.blob],
summary: "Selects the media Artifact this element inspects and normalizes." },
- { name: "video", kind: "literal", required: true,
+ { name: "recipe", kind: "reference", required: false, accepts: [svsRecipeType],
+ summary: "Selects a Recipe containing video, audio and span-authority stream policy." },
+ { name: "video", kind: "literal", required: false,
summary: "Decides which moving-image stream is carried: `primary-moving`, `none`, or `stream:<index>`." },
- { name: "audio", kind: "literal", required: true,
+ { name: "audio", kind: "literal", required: false,
summary: "Decides which audio stream is carried: `default`, `none`, or `stream:<index>`." },
- { name: "span-authority", kind: "literal", required: true,
+ { name: "span-authority", kind: "literal", required: false,
values: ["video", "audio"],
summary: "Decides which selected stream defines the extent the other is trimmed or padded to." },
- { name: "frame-rate", kind: "literal", required: true,
- summary: "Decides the exact rational frame rate of the common frame domain, such as `30` or `30000/1001`." },
+ { name: "clock", kind: "reference", required: false, accepts: [programSpaceTypes.clock],
+ summary: "Selects the authored frame clock shared with the programme." },
+ { name: "frame-rate", kind: "literal", required: false,
+ summary: "Legacy inline frame rate; write exactly one of clock or frame-rate." },
],
ports: [
{ name: "media", type: mediaTypes.synchronized,
summary: "The normalized SynchronizedMedia, addressed as `<id>.media`." },
],
example: `<pipeline:Normalize id="music-media" source={music}
- video="none" audio="default" span-authority="audio" frame-rate="30"/>`,
+ recipe={recipes.media.audio} clock={clock}/>`,
notes: [
- "All six attributes are required; the element accepts no children and no text content.",
+ "Write exactly one of recipe or the direct video/audio/span-authority attributes, and exactly one of clock or frame-rate.",
"`primary-moving` excludes attached-picture streams, prefers one declared default and fails closed on an ambiguous container; `stream:<index>` is for a container the author genuinely knows.",
"Selecting embedded audio is a media fact only and makes no SpeechBasis, speaker or alignment claim.",
],
@@ -236,25 +239,16 @@ export const mediaPipelineMarkupSurfaces = [
},
{
name: "transform-media", tag: "Transform", mode: "structured",
- outputs: [mediaPipelineTypes.selectionRequest, mediaPipelineTypes.transformProgram, artifactTypes.blob],
+ outputs: [mediaPipelineTypes.transformProgram, artifactTypes.blob],
vocabulary: {
summary:
- "Normalizes one BlobArtifact and runs an ordered trim and retime program over it, publishing the transformed video Artifact.",
+ "Runs an ordered trim and retime program over one already normalized SynchronizedMedia value.",
attributes: [
{ name: "id", kind: "identifier", required: true,
- summary: "Names the selection Record, the transform program Record and the transformed video this element publishes." },
+ summary: "Names the transform program Record and the transformed video this element publishes." },
{ name: "source", kind: "reference", required: true,
- accepts: [artifactTypes.blob],
- summary: "Selects the media Artifact this element transforms." },
- { name: "video", kind: "literal", required: true,
- summary: "Decides which moving-image stream is transformed: `primary-moving` or `stream:<index>`." },
- { name: "audio", kind: "literal", required: true,
- summary: "Decides which audio stream travels with the transform: `default`, `none`, or `stream:<index>`." },
- { name: "span-authority", kind: "literal", required: true,
- values: ["video"],
- summary: "Decides which selected stream defines the extent, and a transform is always authored against video." },
- { name: "frame-rate", kind: "literal", required: true,
- summary: "Decides the exact rational frame rate the transform runs on, such as `30` or `30000/1001`." },
+ accepts: [mediaTypes.synchronized],
+ summary: "Selects the prepared SynchronizedMedia this element transforms." },
],
children: [
{ tag: "Trim", cardinality: "many",
@@ -281,15 +275,14 @@ export const mediaPipelineMarkupSurfaces = [
{ name: "video", type: artifactTypes.blob,
summary: "The transformed media Artifact, addressed as `<id>.video`." },
],
- example: `<media:Transform id="prepared" source={shot.video}
- video="primary-moving" audio="default" span-authority="video" frame-rate="30">
+ example: `<media:Transform id="prepared" source={shot-media.media}>
<media:Trim tail="0.25s"/>
<media:Retime rate="1.05" pitch="preserve"/>
</media:Transform>`,
notes: [
"At least one `Trim` or `Retime` child is required, and children run in the order they are written.",
"`Trim` cannot combine `end` and `tail`, and both children are written empty.",
- "`video` cannot be `none`, and the operations are author meaning rather than an arbitrary FFmpeg string.",
+ "Normalization and stream selection are explicit upstream operations rather than hidden Transform policy.",
],
},
},
@@ -359,6 +352,7 @@ export const mediaPipelineManifest: ModuleManifest = {
speechDependency,
programSpaceDependency,
compositionDependency,
+ { module: svsModuleRef },
],
types: [
{
@@ -388,18 +382,6 @@ export const mediaPipelineManifest: ModuleManifest = {
{ name: mediaPipelineCapabilities.mux.name, returns: mediaTypes.muxed },
],
producers: [
- {
- name: mediaPipelineProducers.bindVisualRequest.name,
- inputs: [{ name: "space", type: programSpaceTypes.programSpace }],
- outputs: [{ name: "request", type: mediaPipelineTypes.selectionRequest }],
- needs: [],
- },
- {
- name: mediaPipelineProducers.bindAvRequest.name,
- inputs: [{ name: "space", type: programSpaceTypes.programSpace }],
- outputs: [{ name: "request", type: mediaPipelineTypes.selectionRequest }],
- needs: [],
- },
{
name: mediaPipelineProducers.inspect.name,
inputs: [{ name: "source", type: artifactTypes.blob }],
@@ -480,7 +462,7 @@ export const mediaPipelineManifest: ModuleManifest = {
},
{
name: mediaPipelineProducers.projectSpeechEvidenceAudio.name,
- inputs: [{ name: "audio", type: speechTypes.audioBasis }],
+ inputs: [{ name: "media", type: mediaTypes.synchronized }],
outputs: [],
needs: [{
name: "evidenceAudio",
diff --git a/packages/media-pipeline/src/selection.ts b/packages/media-pipeline/src/selection.ts
index 10f3fa82..a602a485 100644
--- a/packages/media-pipeline/src/selection.ts
+++ b/packages/media-pipeline/src/selection.ts
@@ -1,7 +1,5 @@
-import type { Narrative } from "@hypit/narrative";
import { sealMediaStreamSelection, verifyMediaInspection } from "@hypit/media";
import type { MediaAudioStream, MediaInspection, MediaStreamSelection, MediaVideoStream } from "@hypit/media";
-import type { SpeechBasis } from "@hypit/speech";
import {
canonicalize,
} from "@hypit/protocol";
diff --git a/packages/media-pipeline/src/surface.ts b/packages/media-pipeline/src/surface.ts
index 072ae675..6eb9b9d3 100644
--- a/packages/media-pipeline/src/surface.ts
+++ b/packages/media-pipeline/src/surface.ts
@@ -1,5 +1,17 @@
+import {
+ assertEmptyElement as empty,
+ localName,
+ textAttribute as text,
+ type StructuredElement,
+ type StructuredSurfaceHandler,
+ type SurfaceResolvedReference,
+ type MarkupAttributeValue,
+} from "@hypit/markup";
+import { sameType, type CanonicalValue } from "@hypit/protocol";
import { artifactTypes } from "@hypit/artifact";
-import type { StructuredElement, StructuredSurfaceHandler, SurfaceResolvedReference, MarkupAttributeValue } from "@hypit/markup";
+import { mediaTypes } from "@hypit/media";
+import { programSpaceTypes, type ProgramClock } from "@hypit/program-space";
+import { svsRecipeType, type SvsRecipe } from "@hypit/svs";
import {
extractAudioFragment,
@@ -21,16 +33,6 @@ import type {
MediaTransformOperation,
} from "./types.js";
-function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
- return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
-}
-
-function text(element: StructuredElement, name: string): string {
- const value = element.attributes[name];
- if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${element.name}.${name} must be non-empty text.`);
- return value.trim();
-}
-
function ref(
raw: MarkupAttributeValue | undefined,
label: string,
@@ -42,6 +44,39 @@ function ref(
return value;
}
+function typedReference(
+ raw: MarkupAttributeValue | undefined,
+ label: string,
+ expected: SurfaceResolvedReference["type"],
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ if (typeof raw !== "object" || raw.kind !== "reference") throw new Error(`${label} must be a reference.`);
+ const value = resolve(raw.path);
+ if (value === undefined || !sameType(value.type, expected)) throw new Error(`${label} has the wrong Type.`);
+ return value;
+}
+
+function authoredRecipe(
+ raw: MarkupAttributeValue | undefined,
+ label: string,
+ resolve: (path: string) => SurfaceResolvedReference | undefined,
+): SvsRecipe {
+ const value = typedReference(raw, label, svsRecipeType, resolve);
+ if (value.record?.value.kind !== "inline") throw new Error(`${label} must be an authored SVS Recipe.`);
+ const recipe = value.record.value.value as unknown as SvsRecipe;
+ const unknown = Object.keys(recipe.properties).filter((name) => !["audio", "span-authority", "video"].includes(name));
+ if (unknown.length > 0) throw new Error(`${label} only accepts video, audio and span-authority; found ${unknown.join(", ")}.`);
+ return recipe;
+}
+
+function recipeText(recipe: SvsRecipe, name: string, label: string): string {
+ const value: CanonicalValue | undefined = recipe.properties[name];
+ if (typeof value !== "string" || value.trim().length === 0) {
+ throw new Error(`${label} requires string property ${name}.`);
+ }
+ return value;
+}
+
function stream(value: string, kind: "video"): MediaSelectionRequest["video"];
function stream(value: string, kind: "audio"): MediaSelectionRequest["audio"];
function stream(value: string, kind: "video" | "audio"): MediaSelectionRequest["video"] | MediaSelectionRequest["audio"] {
@@ -64,10 +99,6 @@ function frameRate(value: string): { readonly numerator: number; readonly denomi
return { numerator, denominator };
}
-function localName(value: string): string {
- return value.includes(":") ? value.slice(value.lastIndexOf(":") + 1) : value;
-}
-
function exactAttributes(element: StructuredElement, required: readonly string[], optional: readonly string[] = []): void {
const expected = new Set([...required, ...optional]);
const unknown = Object.keys(element.attributes).filter((name) => !expected.has(name));
@@ -101,12 +132,6 @@ function videoSelector(value: string): FrameExtractionRequest["video"] {
return { mode: "stream-index", streamIndex: Number(match[1]) };
}
-function empty(element: StructuredElement): void {
- if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
- throw new Error(`${element.name} must be empty.`);
- }
-}
-
function transformOperations(element: StructuredElement): readonly MediaTransformOperation[] {
const result: MediaTransformOperation[] = [];
for (const child of element.children) {
@@ -151,24 +176,41 @@ function transformOperations(element: StructuredElement): readonly MediaTransfor
}
export const decodeSynchronizedMediaSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- const expected = ["audio", "frame-rate", "id", "source", "span-authority", "video"];
- if (Object.keys(element.attributes).sort().join("\0") !== expected.sort().join("\0")) {
- throw new Error(`${element.name} requires exactly id, source, video, audio, span-authority and frame-rate.`);
+ const required = ["id", "source"];
+ const optional = ["audio", "clock", "frame-rate", "recipe", "span-authority", "video"];
+ exactAttributes(element, required, optional);
+ const hasClock = element.attributes.clock !== undefined;
+ const hasFrameRate = element.attributes["frame-rate"] !== undefined;
+ if (hasClock === hasFrameRate) throw new Error(`${element.name} requires exactly one of clock or frame-rate.`);
+ const hasRecipe = element.attributes.recipe !== undefined;
+ const policyCount = ["video", "audio", "span-authority"].filter((name) => element.attributes[name] !== undefined).length;
+ if ((hasRecipe && policyCount !== 0) || (!hasRecipe && policyCount !== 3)) {
+ throw new Error(`${element.name} requires exactly one of recipe or video/audio/span-authority.`);
}
if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
throw new Error(`${element.name} must be empty.`);
}
const id = text(element, "id");
const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
- const video = stream(text(element, "video"), "video");
- const audio = stream(text(element, "audio"), "audio");
- const spanAuthority = text(element, "span-authority");
+ const clock = hasClock
+ ? typedReference(element.attributes.clock, `${element.name}.clock`, programSpaceTypes.clock, resolveReference)
+ : undefined;
+ const clockValue = clock?.record?.value.kind === "inline"
+ ? clock.record.value.value as unknown as ProgramClock
+ : undefined;
+ if (clock !== undefined && clockValue === undefined) throw new Error(`${element.name}.clock must be an authored Clock record.`);
+ const normalizationRecipe = hasRecipe ? authoredRecipe(element.attributes.recipe, `${element.name}.recipe`, resolveReference) : undefined;
+ const video = stream(normalizationRecipe === undefined ? text(element, "video") : recipeText(normalizationRecipe, "video", `${element.name}.recipe`), "video");
+ const audio = stream(normalizationRecipe === undefined ? text(element, "audio") : recipeText(normalizationRecipe, "audio", `${element.name}.recipe`), "audio");
+ const spanAuthority = normalizationRecipe === undefined
+ ? text(element, "span-authority")
+ : recipeText(normalizationRecipe, "span-authority", `${element.name}.recipe`);
if (spanAuthority !== "video" && spanAuthority !== "audio") throw new Error(`${element.name}.span-authority must be video or audio.`);
const request = sealMediaSelectionRequest({
video,
audio,
spanAuthority,
- frameRate: frameRate(text(element, "frame-rate")),
+ frameRate: clockValue?.frameRate ?? frameRate(text(element, "frame-rate")),
});
const requestId = `${id}.request`;
return {
@@ -185,36 +227,22 @@ export const decodeSynchronizedMediaSurface: StructuredSurfaceHandler = ({ eleme
};
export const decodeTransformMediaSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- exactAttributes(element, ["id", "source", "video", "audio", "span-authority", "frame-rate"]);
+ exactAttributes(element, ["id", "source"]);
const id = text(element, "id");
- const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
- const video = stream(text(element, "video"), "video");
- if (video.mode === "none") throw new Error(`${element.name}.video cannot be none.`);
- const audio = stream(text(element, "audio"), "audio");
- const spanAuthority = text(element, "span-authority");
- if (spanAuthority !== "video") throw new Error(`${element.name}.span-authority must be video.`);
- const selection = sealMediaSelectionRequest({
- video,
- audio,
- spanAuthority,
- frameRate: frameRate(text(element, "frame-rate")),
- });
+ const source = typedReference(element.attributes.source, `${element.name}.source`, mediaTypes.synchronized, resolveReference);
const program = sealMediaTransformProgram({
operations: transformOperations(element),
});
- const selectionId = `${id}.selection`;
const programId = `${id}.program`;
return {
records: [
- { id: selectionId, type: mediaPipelineTypes.selectionRequest, value: { kind: "inline", value: selection }, range: element.range },
{ id: programId, type: mediaPipelineTypes.transformProgram, value: { kind: "inline", value: program }, range: element.range },
],
components: [{
id,
fragment: transformMediaFragment.id,
inputs: {
- source: source.ref,
- selection: { kind: "record", id: selectionId },
+ media: source.ref,
program: { kind: "record", id: programId },
},
outputs: { video: `${id}.video` },
diff --git a/packages/media-pipeline/test/manifest.test.ts b/packages/media-pipeline/test/manifest.test.ts
index 8a013f50..d77cb8c3 100644
--- a/packages/media-pipeline/test/manifest.test.ts
+++ b/packages/media-pipeline/test/manifest.test.ts
@@ -1,4 +1,7 @@
import { artifactTypes } from "@hypit/artifact";
+import { mediaTypes } from "@hypit/media";
+import { programSpaceTypes } from "@hypit/program-space";
+import { svsRecipeType } from "@hypit/svs";
import assert from "node:assert/strict";
import test from "node:test";
@@ -24,17 +27,21 @@ test("the Normalize Surface makes inspection and normalization an explicit autho
attributes: {
id: "motion",
source: { kind: "reference", path: "generated" },
- video: "primary-moving",
- audio: "none",
- "span-authority": "video",
- "frame-rate": "30000/1001",
+ clock: { kind: "reference", path: "clock" },
+ recipe: { kind: "reference", path: "policy" },
},
children: [],
range,
},
resolveReference: (path) => path === "generated"
? { path, ref: { kind: "record", id: path }, type: artifactTypes.blob }
- : undefined,
+ : path === "clock"
+ ? { path, ref: { kind: "record", id: path }, type: programSpaceTypes.clock,
+ record: { value: { kind: "inline", value: { frameRate: { numerator: 30_000, denominator: 1_001 } } } } as never }
+ : path === "policy"
+ ? { path, ref: { kind: "record", id: path }, type: svsRecipeType,
+ record: { value: { kind: "inline", value: { path, properties: { video: "primary-moving", audio: "none", "span-authority": "video" } } } } as never }
+ : undefined,
resolveAsset: async () => { throw new Error("no asset resolution expected"); },
});
assert.equal(result.fragments[0]?.id, synchronizedMediaFragment.id);
@@ -51,9 +58,11 @@ test("the Normalize Surface makes inspection and normalization an explicit autho
test("media operations are ordinary graph branches over BlobArtifact", async () => {
const range = { source: "operations.svml", start: 0, end: 1 };
- const resolveReference = (path: string) => path === "generated.video"
- ? { path, ref: { kind: "record" as const, id: path }, type: artifactTypes.blob }
- : undefined;
+ const resolveReference = (path: string) => path === "generated.media"
+ ? { path, ref: { kind: "record" as const, id: path }, type: mediaTypes.synchronized }
+ : path === "generated.video"
+ ? { path, ref: { kind: "record" as const, id: path }, type: artifactTypes.blob }
+ : undefined;
const context = { sourceName: range.source, resolveReference,
resolveAsset: async () => { throw new Error("no asset resolution expected"); } };
const transform = await decodeTransformMediaSurface({
@@ -61,8 +70,7 @@ test("media operations are ordinary graph branches over BlobArtifact", async ()
element: {
kind: "element", name: "media:Transform", range,
attributes: {
- id: "prepared", source: { kind: "reference", path: "generated.video" },
- video: "primary-moving", audio: "default", "span-authority": "video", "frame-rate": "30",
+ id: "prepared", source: { kind: "reference", path: "generated.media" },
},
children: [
{ kind: "element", name: "media:Trim", attributes: { tail: "0.25s" }, children: [], range },
diff --git a/packages/media-track/README.md b/packages/media-track/README.md
index d00c36ed..fd1e9aa9 100644
--- a/packages/media-track/README.md
+++ b/packages/media-track/README.md
@@ -5,10 +5,9 @@ Official provider-free Media Item and Sequence authoring package.
`Track` accepts explicit ProgramSpace, CanvasSpace and, when semantic points are used, a
CompleteSemanticMap. Each independently timed `Item` receives an explicit SpatialFrame and either
one direct source or ordered Paint/sample layers. A visual source is always named by its actual
-form: `image`, generated/raw `video`, prepared `media`, or compositable `surface`. For raw video the
-Track Surface declares the connected ProgramSpace as its consumption policy and expands ordinary
-Media Pipeline inspect/select/normalize Operations; it does not execute normalization itself.
-`audio="include"` opts audio into that same graph. `Sequence` owns an ordered replacement surface with explicit
+form: `image`, prepared `media`, or compositable `surface`. Moving video must be prepared by an
+explicit Media Pipeline `Normalize` Operation before it enters the Track; the Track never guesses a
+stream, frame rate or audio policy. `Sequence` owns an ordered replacement surface with explicit
activation points and pairwise Handoffs. Package-owned Recipes cover fitting, source occupancy,
frame Paint, clipping, borders, shadows, lifecycle motion and sampling motion.
@@ -17,7 +16,5 @@ Visual and audio are separate deterministic projections. A visual-only Track exp
exports a peer `AudioTrack`. Speech Spine reuses the same lowering laws through a deliberately
restricted internal projection.
-The automatic lowering is Surface behavior, not a privileged Runtime route: it produces the same
-ordinary Media Pipeline Operations that an author can write explicitly through
-`@hypit/media-pipeline`. The package does not add Media, B-roll or Provider meaning to Core,
+The package lowers only track composition. It does not add Media, B-roll or Provider meaning to Core,
Film, Composition or HyperFrames. “B-roll” is an editorial use of an ordinary Item or Sequence.
diff --git a/packages/media-track/package.json b/packages/media-track/package.json
index 9d91392a..d2ddd7d8 100644
--- a/packages/media-track/package.json
+++ b/packages/media-track/package.json
@@ -16,7 +16,6 @@
"@hypit/composition": "workspace:*",
"@hypit/elaborator": "workspace:*",
"@hypit/media": "workspace:*",
- "@hypit/media-pipeline": "workspace:*",
"@hypit/narrative": "workspace:*",
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
diff --git a/packages/media-track/src/manifest.ts b/packages/media-track/src/manifest.ts
index 12cb8c54..12e473e9 100644
--- a/packages/media-track/src/manifest.ts
+++ b/packages/media-track/src/manifest.ts
@@ -9,7 +9,6 @@ import {
} from "@hypit/media";
import { narrativeDependency, narrativeTypes } from "@hypit/narrative";
import { programSpaceDependency, programSpaceTypes } from "@hypit/program-space";
-import { mediaPipelineManifest, mediaPipelineModuleRef } from "@hypit/media-pipeline";
import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/protocol";
import { semanticMapDependency, semanticMapTypes } from "@hypit/semantic-map";
import {
@@ -597,7 +596,6 @@ export const mediaTrackManifest: ModuleManifest = {
dependencies: [
artifactDependency,
mediaDependency,
- { module: mediaPipelineModuleRef },
narrativeDependency,
semanticMapDependency,
programSpaceDependency,
diff --git a/packages/media-track/src/surface.ts b/packages/media-track/src/surface.ts
index 3e9bb981..71797d58 100644
--- a/packages/media-track/src/surface.ts
+++ b/packages/media-track/src/surface.ts
@@ -1,23 +1,26 @@
+import {
+ assertEmptyElement as empty,
+ assertAttributes as allowed,
+ textAttribute as text,
+ optionalTextAttribute as optionalText,
+ type StructuredElement,
+ type StructuredSurfaceHandler,
+ type SurfaceRecordDraft,
+ type SurfaceResolvedReference,
+ type MarkupAttributeValue,
+} from "@hypit/markup";
+import { sameType, type TypeRef } from "@hypit/protocol";
import { artifactTypes } from "@hypit/artifact";
import { compositionTypes } from "@hypit/composition";
import { sealGraphFragment } from "@hypit/elaborator";
import type { FragmentOperation } from "@hypit/elaborator";
import { mediaTypes } from "@hypit/media";
-import { mediaPipelineProducers } from "@hypit/media-pipeline";
import { narrativeTypes } from "@hypit/narrative";
import { programSpaceTypes } from "@hypit/program-space";
-import type { TypeRef } from "@hypit/protocol";
import { semanticMapTypes } from "@hypit/semantic-map";
import { spatialTypes } from "@hypit/spatial";
import { svsRecipeType } from "@hypit/svs";
import type { SvsRecipe } from "@hypit/svs";
-import type {
- StructuredElement,
- StructuredSurfaceHandler,
- SurfaceRecordDraft,
- SurfaceResolvedReference,
- MarkupAttributeValue,
-} from "@hypit/markup";
import type {
OccurrenceExpansion,
TemporalDuration,
@@ -52,42 +55,6 @@ import type {
const input = (name: string) => ({ kind: "fragment-input" as const, name });
const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
-function sameType(left: TypeRef, right: TypeRef): boolean {
- return left.module.name === right.module.name
- && left.module.version === right.module.version
- && left.name === right.name;
-}
-
-function allowed(element: StructuredElement, names: readonly string[]): void {
- const permit = new Set(names);
- const unknown = Object.keys(element.attributes).filter((name) => !permit.has(name));
- if (unknown.length > 0) throw new Error(`${element.name} has unsupported attributes ${unknown.join(", ")}.`);
-}
-
-function empty(element: StructuredElement): void {
- if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
- throw new Error(`${element.name} must be empty.`);
- }
-}
-
-function text(element: StructuredElement, name: string, fallback?: string): string {
- const value = element.attributes[name];
- if (value === undefined && fallback !== undefined) return fallback;
- if (typeof value !== "string" || value.trim().length === 0) {
- throw new Error(`${element.name}.${name} must be non-empty text.`);
- }
- return value.trim();
-}
-
-function optionalText(element: StructuredElement, name: string): string | undefined {
- const value = element.attributes[name];
- if (value === undefined) return undefined;
- if (typeof value !== "string" || value.trim().length === 0) {
- throw new Error(`${element.name}.${name} must be non-empty text.`);
- }
- return value.trim();
-}
-
function numberValue(element: StructuredElement, name: string, fallback?: number): number {
const raw = optionalText(element, name);
if (raw === undefined && fallback !== undefined) return fallback;
@@ -240,7 +207,6 @@ function temporalBinding(
type FragmentLayer =
| { readonly kind: "paint"; readonly specName: string }
| { readonly kind: "still"; readonly sourceName: string; readonly extentName: string; readonly fitName: string; readonly specName: string }
- | { readonly kind: "video"; readonly sourceName: string; readonly includeAudio: boolean; readonly fitName: string; readonly specName: string }
| { readonly kind: "timed" | "surface"; readonly sourceName: string; readonly fitName: string; readonly specName: string };
type FragmentSound = { readonly sourceName: string; readonly specName: string };
@@ -288,46 +254,6 @@ function appendLayers(operations: FragmentOperation[], prefix: string, layers: r
operations.push({ id, producer: mediaTrackProducers.appendStillLayer, inputs: {
...common, source: input(layer.sourceName), extent: input(layer.extentName), fit: input(layer.fitName),
}, result: { kind: "output", name: "layers" } });
- } else if (layer.kind === "video") {
- const requestId = `${id}:request`;
- const inspectId = `${id}:inspect`;
- const selectId = `${id}:select`;
- const normalizeId = `${id}:normalize`;
- operations.push(
- {
- id: requestId,
- producer: layer.includeAudio ? mediaPipelineProducers.bindAvRequest : mediaPipelineProducers.bindVisualRequest,
- inputs: { space: input("space") },
- result: { kind: "output", name: "request" },
- },
- {
- id: inspectId,
- producer: mediaPipelineProducers.inspect,
- inputs: { source: input(layer.sourceName) },
- result: { kind: "need", name: "inspection" },
- },
- {
- id: selectId,
- producer: mediaPipelineProducers.select,
- inputs: { inspection: operation(inspectId), request: operation(requestId) },
- result: { kind: "output", name: "selection" },
- },
- {
- id: normalizeId,
- producer: mediaPipelineProducers.normalize,
- inputs: {
- source: input(layer.sourceName), inspection: operation(inspectId),
- selection: operation(selectId), request: operation(requestId),
- },
- result: { kind: "need", name: "media" },
- },
- {
- id,
- producer: mediaTrackProducers.appendTimedLayer,
- inputs: { ...common, source: operation(normalizeId), fit: input(layer.fitName) },
- result: { kind: "output", name: "layers" },
- },
- );
} else {
operations.push({ id, producer: layer.kind === "timed" ? mediaTrackProducers.appendTimedLayer : mediaTrackProducers.appendSurfaceLayer,
inputs: { ...common, source: input(layer.sourceName), fit: input(layer.fitName) }, result: { kind: "output", name: "layers" } });
@@ -485,11 +411,10 @@ type SourceLayerContext = {
type DeclaredVisualSource =
| { readonly kind: "image"; readonly value: SurfaceResolvedReference; readonly extent: SurfaceResolvedReference }
- | { readonly kind: "video"; readonly value: SurfaceResolvedReference; readonly includeAudio: boolean }
| { readonly kind: "media"; readonly value: SurfaceResolvedReference }
| { readonly kind: "surface"; readonly value: SurfaceResolvedReference };
-const VISUAL_SOURCE_ATTRIBUTES = ["image", "video", "media", "surface"] as const;
+const VISUAL_SOURCE_ATTRIBUTES = ["image", "media", "surface"] as const;
function declaredVisualSource(
element: StructuredElement,
@@ -499,14 +424,14 @@ function declaredVisualSource(
const present = VISUAL_SOURCE_ATTRIBUTES.filter((name) => element.attributes[name] !== undefined);
if (present.length === 0) {
if (element.attributes.extent !== undefined) throw new Error(`${element.name}.extent requires image={Artifact}.`);
- if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio requires video={Artifact}.`);
- if (required) throw new Error(`${element.name} requires exactly one of image, video, media or surface.`);
+ if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
+ if (required) throw new Error(`${element.name} requires exactly one of image, media or surface.`);
return undefined;
}
- if (present.length !== 1) throw new Error(`${element.name} requires exactly one of image, video, media or surface.`);
+ if (present.length !== 1) throw new Error(`${element.name} requires exactly one of image, media or surface.`);
const kind = present[0]!;
if (kind === "image") {
- if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is only valid with video={Artifact}.`);
+ if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
return {
kind,
value: reference(element.attributes.image, `${element.name}.image`, artifactTypes.blob, resolve),
@@ -514,16 +439,7 @@ function declaredVisualSource(
};
}
if (element.attributes.extent !== undefined) throw new Error(`${element.name}.extent is only valid with image={Artifact}.`);
- if (kind === "video") {
- const audio = optionalText(element, "audio") ?? "omit";
- if (audio !== "include" && audio !== "omit") throw new Error(`${element.name}.audio must be include or omit.`);
- return {
- kind,
- value: reference(element.attributes.video, `${element.name}.video`, artifactTypes.blob, resolve),
- includeAudio: audio === "include",
- };
- }
- if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is only valid with video={Artifact}.`);
+ if (element.attributes.audio !== undefined) throw new Error(`${element.name}.audio is no longer accepted; normalize the source explicitly and use media={...}.`);
return kind === "media"
? { kind, value: reference(element.attributes.media, `${element.name}.media`, mediaTypes.synchronized, resolve) }
: { kind, value: reference(element.attributes.surface, `${element.name}.surface`, mediaTypes.compositableSurface, resolve) };
@@ -589,7 +505,6 @@ function sourceLayer(
state.addReference(extentName, (source as Extract<DeclaredVisualSource, { readonly kind: "image" }>).extent);
return { kind: "still", sourceName, extentName, fitName, specName };
}
- if (source.kind === "video") return { kind: "video", sourceName, includeAudio: source.includeAudio, fitName, specName };
return { kind: sourceKind, sourceName, fitName, specName };
}
@@ -642,7 +557,7 @@ function unitLayers(
layers.push({ kind: "paint", specName: name });
continue;
}
- allowed(child, ["id", "image", "video", "media", "surface", "extent", "audio", "appearance"]);
+ allowed(child, ["id", "image", "media", "surface", "extent", "appearance"]);
layers.push(sourceLayer(state, {
trackId: input.trackId, unitSuffix: input.unitSuffix, defaultRecipe: input.appearance, sourceElement: child,
}, layerIndex, resolve));
@@ -668,13 +583,8 @@ function validateUnitChildren(element: StructuredElement, direct: boolean, allow
function sourceAudio(
element: StructuredElement,
- directSource?: DeclaredVisualSource,
): MediaSequenceMemberSpec["sourceAudio"] {
const fromLayer = optionalText(element, "source-audio");
- if (directSource?.kind === "video" && directSource.includeAudio) {
- if (fromLayer !== undefined) throw new Error(`${element.name} cannot combine audio=include with source-audio.`);
- return { fromLayer: "content", gain: numberValue(element, "audio-gain", 1) };
- }
if (fromLayer === undefined) {
if (element.attributes["audio-gain"] !== undefined) throw new Error(`${element.name}.audio-gain requires source-audio.`);
return undefined;
@@ -740,7 +650,7 @@ export const decodeMediaTrackSurface: StructuredSurfaceHandler = ({ element, res
itemIndex += 1;
const itemSuffix = suffix(itemIndex);
allowed(child, [
- "id", "image", "video", "media", "surface", "extent", "audio", "frame", "appearance", "motion", "source-audio", "audio-gain",
+ "id", "image", "media", "surface", "extent", "frame", "appearance", "motion", "source-audio", "audio-gain",
"clip", "during", "at", "for", "start", "end", "selection", "segment", "moment", "occurrences",
]);
const id = optionalText(child, "id") ?? `${trackId}.item.${itemSuffix}`;
@@ -761,7 +671,7 @@ export const decodeMediaTrackSurface: StructuredSurfaceHandler = ({ element, res
validateUnitChildren(child, directSource !== undefined, true);
const layers = unitLayers(state, { trackId, unitSuffix: `item-${itemSuffix}`, element: child, appearance,
...(directSource === undefined ? {} : { directSource }), allowFramePaint: true }, resolveReference);
- const selectedAudio = sourceAudio(child, directSource);
+ const selectedAudio = sourceAudio(child);
const specName = `item-${itemSuffix}-spec`;
state.addRecord(specName, `${trackId}.item.${itemSuffix}.spec`, mediaTrackTypes.itemSpec,
decodeMediaItemSpec(appearance, { id, projection: binding.projection, expansion,
@@ -815,7 +725,7 @@ export const decodeMediaTrackSurface: StructuredSurfaceHandler = ({ element, res
const members: FragmentMember[] = [];
for (const [index, member] of memberElements.entries()) {
const memberSuffix = suffix(index + 1);
- allowed(member, ["id", "image", "video", "media", "surface", "extent", "audio", "appearance", "at", "boundary", "source-audio", "audio-gain"]);
+ allowed(member, ["id", "image", "media", "surface", "extent", "appearance", "at", "boundary", "source-audio", "audio-gain"]);
const at = oneOfReferences(member.attributes.at, `${member.name}.at`, [narrativeTypes.moment, narrativeTypes.selection], resolveReference);
usesMap = true;
const boundary = optionalText(member, "boundary");
@@ -830,7 +740,7 @@ export const decodeMediaTrackSurface: StructuredSurfaceHandler = ({ element, res
const unitSuffix = `sequence-${sequenceSuffix}-member-${memberSuffix}`;
const layers = unitLayers(state, { trackId, unitSuffix, element: member, appearance: memberAppearance,
...(directSource === undefined ? {} : { directSource }), allowFramePaint: true }, resolveReference);
- const selectedAudio = sourceAudio(member, directSource);
+ const selectedAudio = sourceAudio(member);
if (selectedAudio !== undefined) hasAudio = true;
const specName = `${unitSuffix}-spec`;
state.addRecord(specName, `${trackId}.${unitSuffix}.spec`, mediaTrackTypes.memberSpec,
diff --git a/packages/media-track/test/media-track.test.ts b/packages/media-track/test/media-track.test.ts
index 23ca0fde..d093e3ac 100644
--- a/packages/media-track/test/media-track.test.ts
+++ b/packages/media-track/test/media-track.test.ts
@@ -7,7 +7,6 @@ import type { AudioTrack } from "@hypit/composition";
import { compileHyperframesDocument } from "@hypit/hyperframes";
import { mediaTypes } from "@hypit/media";
import type { CompositableSurfaceRef, SynchronizedMedia } from "@hypit/media";
-import { mediaPipelineProducers } from "@hypit/media-pipeline";
import { artifactTypes } from "@hypit/artifact";
import { narrativeTypes } from "@hypit/narrative";
import {
@@ -692,7 +691,6 @@ test("the Media author Surface emits explicit graph edges for layers, semantic t
["frame", plain("frame", spatialTypes.frame)],
["clip-path", plain("clip-path", spatialTypes.path)],
["still", plain("still", artifactTypes.blob)],
- ["raw-video", plain("raw-video", artifactTypes.blob)],
["extent", plain("extent", spatialTypes.extent)],
["video", plain("video", mediaTypes.synchronized)],
["surface", plain("surface", mediaTypes.compositableSurface)],
@@ -716,7 +714,7 @@ test("the Media author Surface emits explicit graph edges for layers, semantic t
node("media:Item", { id: "segment-card", image: ref("still"), extent: ref("extent"), frame: ref("frame"), appearance: ref("still-style"), during: ref("answer-segment") }),
node("media:Item", { id: "proof", frame: ref("frame"), appearance: ref("card-style"), motion: ref("motion"), during: ref("selection"), "source-audio": "video", "audio-gain": "0.8" }, [
node("media:Paint", { id: "backing", appearance: ref("paint-style") }),
- node("media:Layer", { id: "video", video: ref("raw-video"), audio: "include", appearance: ref("video-style") }, [
+ node("media:Layer", { id: "video", media: ref("video"), appearance: ref("video-style") }, [
node("media:Sampling", { at: "start", zoom: "1" }),
node("media:Sampling", { at: "end", zoom: "1.1", y: "-10", easing: "ease-out" }),
]),
@@ -742,9 +740,6 @@ test("the Media author Surface emits explicit graph edges for layers, semantic t
assert.ok(producers.includes("append-still-media-layer"));
assert.ok(producers.includes("append-segment-media-item"));
assert.ok(producers.includes("append-timed-media-layer"));
- assert.ok(producers.includes(mediaPipelineProducers.bindAvRequest.name));
- assert.ok(producers.includes(mediaPipelineProducers.inspect.name));
- assert.ok(producers.includes(mediaPipelineProducers.normalize.name));
assert.ok(producers.includes("append-surface-media-layer"));
assert.ok(producers.includes("append-media-paint-layer"));
assert.ok(producers.includes("bind-media-item-clip-path"));
diff --git a/packages/program-space/README.md b/packages/program-space/README.md
index cd47480a..3480efe8 100644
--- a/packages/program-space/README.md
+++ b/packages/program-space/README.md
@@ -1,3 +1,7 @@
# `@hypit/program-space`
The provider-neutral time and frame domain shared by a compiled program.
+
+`Clock` is the authored, duration-free frame-rate fact for a video project. It is declared once and
+consumed by media normalization and Speech Spine; `ProgramSpace` is the later duration-bearing
+domain assembled from the semantic Takes.
diff --git a/packages/program-space/package.json b/packages/program-space/package.json
index 2e7923de..34631238 100644
--- a/packages/program-space/package.json
+++ b/packages/program-space/package.json
@@ -11,6 +11,7 @@
"activation": "./src/activation.ts"
},
"dependencies": {
+ "@hypit/markup": "workspace:*",
"@hypit/protocol": "workspace:*"
}
}
diff --git a/packages/program-space/src/activation.ts b/packages/program-space/src/activation.ts
index a360a2b0..d7511bb7 100644
--- a/packages/program-space/src/activation.ts
+++ b/packages/program-space/src/activation.ts
@@ -1,3 +1,15 @@
-import { programSpaceManifest } from "./index.js";
-export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: programSpaceManifest }] };
+import { createMarkupSurfaceHostFacet } from "@hypit/markup";
+
+import { decodeClockSurface } from "./surface.js";
+import { programSpaceManifest, programSpaceMarkupSurfaces, programSpaceModuleRef } from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.node-package@1" as const,
+ modules: [{ manifest: programSpaceManifest }],
+ hostFacets: [createMarkupSurfaceHostFacet({
+ module: programSpaceModuleRef,
+ declaration: programSpaceMarkupSurfaces.find((item) => item.name === "clock")!,
+ handler: decodeClockSurface,
+ })],
+};
export default hypitPackage;
diff --git a/packages/program-space/src/index.ts b/packages/program-space/src/index.ts
index 727a22c0..542d98a6 100644
--- a/packages/program-space/src/index.ts
+++ b/packages/program-space/src/index.ts
@@ -1,5 +1,9 @@
import type { ModuleManifest, TypeRef, ValueSchema } from "@hypit/protocol";
+export type ProgramClock = {
+ readonly frameRate: { readonly numerator: number; readonly denominator: number };
+};
+
export type ProgramSpace = {
readonly durationSec: number;
readonly frameRate: { readonly numerator: number; readonly denominator: number };
@@ -8,6 +12,7 @@ export type ProgramSpace = {
export const programSpaceModuleRef = { name: "@hypit/program-space", version: "1" } as const;
export const programSpaceTypes = {
programSpace: { module: programSpaceModuleRef, name: "ProgramSpace" },
+ clock: { module: programSpaceModuleRef, name: "ProgramClock" },
} satisfies Record<string, TypeRef>;
const number = { kind: "number", minimum: 0 } as const;
const integer = { kind: "number", integer: true, minimum: 0 } as const;
@@ -22,12 +27,29 @@ export const programSpaceSchema: ValueSchema = {
};
export const programSpaceManifest: ModuleManifest = {
format: "hypit.module@1", name: programSpaceModuleRef.name, version: programSpaceModuleRef.version,
- dependencies: [], types: [{
- name: programSpaceTypes.programSpace.name,
- }],
+ dependencies: [], types: [
+ { name: programSpaceTypes.programSpace.name },
+ { name: programSpaceTypes.clock.name },
+ ],
capabilities: [], producers: [],
};
export const programSpaceDependency = { module: programSpaceModuleRef } as const;
+export const programSpaceMarkupSurfaces = [
+ { name: "clock", tag: "Clock", mode: "structured", outputs: [programSpaceTypes.clock] },
+] as const;
+
+export function sealProgramClock(value: ProgramClock): ProgramClock {
+ assertProgramClockIdentity(value);
+ return structuredClone(value);
+}
+
+export function assertProgramClockIdentity(clock: ProgramClock): void {
+ const { numerator, denominator } = clock.frameRate;
+ if (!Number.isSafeInteger(numerator) || numerator <= 0
+ || !Number.isSafeInteger(denominator) || denominator <= 0) {
+ throw new Error("ProgramClock is invalid.");
+ }
+}
export function sealProgramSpace(value: ProgramSpace): ProgramSpace { return structuredClone(value); }
diff --git a/packages/program-space/src/surface.ts b/packages/program-space/src/surface.ts
new file mode 100644
index 00000000..3f0e3999
--- /dev/null
+++ b/packages/program-space/src/surface.ts
@@ -0,0 +1,36 @@
+import {
+ assertEmptyElement as empty,
+ textAttribute as text,
+ type StructuredElement,
+ type StructuredSurfaceHandler,
+} from "@hypit/markup";
+
+import { programSpaceTypes, sealProgramClock, type ProgramClock } from "./index.js";
+
+function frameRate(element: StructuredElement): ProgramClock["frameRate"] {
+ const value = text(element, "frame-rate");
+ const match = /^(\d+)(?:\/(\d+))?$/u.exec(value);
+ if (match === null) throw new Error(`${element.name}.frame-rate must be a positive rational such as 30 or 30000/1001.`);
+ const numerator = Number(match[1]);
+ const denominator = Number(match[2] ?? "1");
+ if (!Number.isSafeInteger(numerator) || numerator < 1
+ || !Number.isSafeInteger(denominator) || denominator < 1) {
+ throw new Error(`${element.name}.frame-rate is invalid.`);
+ }
+ return { numerator, denominator };
+}
+
+export const decodeClockSurface: StructuredSurfaceHandler = ({ element }) => {
+ const expected = ["frame-rate", "id"];
+ if (Object.keys(element.attributes).sort().join("\0") !== expected.join("\0")) {
+ throw new Error(`${element.name} requires exactly id and frame-rate.`);
+ }
+ empty(element);
+ const id = text(element, "id");
+ const clock = sealProgramClock({ frameRate: frameRate(element) });
+ return {
+ records: [{ id, type: programSpaceTypes.clock, value: { kind: "inline", value: clock }, range: element.range }],
+ components: [],
+ fragments: [],
+ };
+};
diff --git a/packages/protocol/src/identity.ts b/packages/protocol/src/identity.ts
index 02a7f365..9b4f8693 100644
--- a/packages/protocol/src/identity.ts
+++ b/packages/protocol/src/identity.ts
@@ -27,3 +27,27 @@ export type CapabilityRef = {
readonly module: ModuleRef;
readonly name: string;
};
+
+export function moduleKey(ref: ModuleRef): string {
+ return `${ref.name}@${ref.version}`;
+}
+
+export function typeKey(ref: TypeRef): string {
+ return `${moduleKey(ref.module)}#${ref.name}`;
+}
+
+export function producerKey(ref: ProducerRef): string {
+ return `${moduleKey(ref.module)}#${ref.name}`;
+}
+
+export function capabilityKey(ref: CapabilityRef): string {
+ return `${moduleKey(ref.module)}#${ref.name}`;
+}
+
+export function sameModule(left: ModuleRef, right: ModuleRef): boolean {
+ return left.name === right.name && left.version === right.version;
+}
+
+export function sameType(left: TypeRef, right: TypeRef): boolean {
+ return sameModule(left.module, right.module) && left.name === right.name;
+}
diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts
index 05d98525..ebf89d95 100644
--- a/packages/protocol/src/index.ts
+++ b/packages/protocol/src/index.ts
@@ -3,6 +3,6 @@ export * from "./error.js";
export { modulePackageAbi } from "./module.js";
export { blobRefObjectSchema } from "./value.js";
export type * from "./build.js";
-export type * from "./identity.js";
+export * from "./identity.js";
export type * from "./module.js";
export type * from "./value.js";
diff --git a/packages/script/src/parser.ts b/packages/script/src/parser.ts
index e7955d97..2bba116c 100644
--- a/packages/script/src/parser.ts
+++ b/packages/script/src/parser.ts
@@ -557,7 +557,7 @@ export function parseScript(
const segmentById = new Map(segments.map((segment) => [segment.id, segment]));
/**
- * Resolve one marker to the exact 2M+2N anchor its affinity names. Token and
+ * Resolve one marker to the exact `2m + 2n` anchor its affinity names. Token and
* Segment cuts are equal citizens: a marker with nothing to its left inside a
* Segment snaps to that Segment's own start, never across into the previous
* Segment, whose end may sit at a different time.
diff --git a/packages/semantic-map/src/location.ts b/packages/semantic-map/src/location.ts
index 6540cf5c..394c106f 100644
--- a/packages/semantic-map/src/location.ts
+++ b/packages/semantic-map/src/location.ts
@@ -26,7 +26,7 @@ export function assertNarrativeMomentIdentity(moment: NarrativeMomentRef): void
}
/**
- * Every marker resolved its affinity to one of the map's 2M+2N anchors while the
+ * Every marker resolves its affinity to one of the map's `2m + 2n` anchors while the
* Script was parsed, where the surrounding structure was known. Locating is a
* lookup: Token cuts and Segment cuts are equal citizens here.
*/
diff --git a/packages/semantic-map/src/types.ts b/packages/semantic-map/src/types.ts
index 282efd19..56602740 100644
--- a/packages/semantic-map/src/types.ts
+++ b/packages/semantic-map/src/types.ts
@@ -6,8 +6,10 @@ export type TimedSpeechToken = {
};
export type SemanticTimePoint = { readonly identity: string; readonly frame: number };
/**
- * The 2M+2N point map every marker addresses, plus the Segment and Token ranges
- * consumers read windows from. Anchors are the addressing truth.
+ * The point map every marker addresses, plus the Segment and Token ranges
+ * consumers read windows from. For `m` Segments and `n` Tokens this contains
+ * `2m + 2n` structural boundaries; one Segment with `k` Tokens therefore has
+ * `2 + 2k` anchors. Anchors are the addressing truth.
*/
export type CompleteSemanticMap = {
readonly tokens: readonly TimedSpeechToken[];
diff --git a/packages/speech-alignment/README.md b/packages/speech-alignment/README.md
index 847984ba..8f8498f9 100644
--- a/packages/speech-alignment/README.md
+++ b/packages/speech-alignment/README.md
@@ -1,28 +1,24 @@
# `@hypit/speech-alignment`
-Provider-neutral deterministic timing locator for one aligned transcript. The package does not
+Provider-neutral deterministic timing locator for one normalized Segment Take. The package does not
invoke Python, read audio, call a speech provider, use an LLM, infer speakers or retain multiple
candidate paths. A runtime adapter preserves its raw provider artifact and supplies the normalized
word, character, score and speech-activity evidence used here in exact 16 kHz sample coordinates.
-Using the explicitly connected SpeechAudioBasis, the locator first projects exact Segment frame
-cuts into the same evidence-sample domain and assigns acoustic passages there. It then aligns each
-known Script Segment independently. A bounded monotonic M:N dynamic
+The locator aligns one explicitly connected Script Segment against the acoustic evidence from that
+same normalized Take. A bounded monotonic M:N dynamic
program accepts exact, split, merge, replacement, source-omission and evidence-insertion groups.
Within a selected group, timed evidence characters determine Script token boundaries; word times
and speech-activity bounds are fallbacks. Missing token runs receive continuous weighted windows
between their measured neighbors instead of invented point timestamps. The result is quantized once
-into the selected ProgramSpace and contains one timing for every Script token and all `2M + 2N`
-semantic anchors. Segment anchors copy the Basis frame cuts directly; they never round-trip through
-floating-point seconds.
+into the Take's local frame domain and contains one timing for every Script token plus the Segment
+and token boundary anchors. Segment anchors are exactly frame `0` and the Take frame count.
-The locator accepts Narrative, SpeechAudioBasis and AlignedTranscriptEvidence through three explicit
-graph edges. Evidence carries no authored Segment identity; the locator checks its sample windows
-against the supplied Basis and performs that association locally.
-Artifact and Basis relationships stay in explicit upstream graph edges instead of being copied
-through every evidence value.
+The public Producer accepts Narrative, one Segment excerpt, `SynchronizedMedia` and
+`AlignedTranscriptEvidence` through explicit graph edges. Evidence carries no authored Segment
+identity; the deterministic semantic step performs that association locally.
-`speechAlignmentComponent` exposes the locator as one enumerable deterministic Producer facet. Its
+`speechAlignmentComponent` exposes this Segment-local projection as one enumerable deterministic Producer facet. Its
identity is checked against `speechAlignmentManifest` and executes through the host-neutral compute
port. The package depends only on public video
contracts and protocol utilities; it has no Core, Driver, Provider, Artifact, queue or credential
diff --git a/packages/speech-alignment/package.json b/packages/speech-alignment/package.json
index 03885ec3..fb987db0 100644
--- a/packages/speech-alignment/package.json
+++ b/packages/speech-alignment/package.json
@@ -12,6 +12,7 @@
},
"dependencies": {
"@hypit/component-kit": "workspace:*",
+ "@hypit/media": "workspace:*",
"@hypit/narrative": "workspace:*",
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
diff --git a/packages/speech-alignment/src/component.ts b/packages/speech-alignment/src/component.ts
index 54456609..a03c20a9 100644
--- a/packages/speech-alignment/src/component.ts
+++ b/packages/speech-alignment/src/component.ts
@@ -1,13 +1,11 @@
import type { ComponentPackage } from "@hypit/component-kit";
-import type { Narrative } from "@hypit/narrative";
-import type { SpeechAudioBasis } from "@hypit/speech";
+import type { SynchronizedMedia } from "@hypit/media";
+import type { Narrative, NarrativeExcerpt } from "@hypit/narrative";
import type { AlignedTranscriptEvidence } from "@hypit/speech-evidence";
import type { StoredValue } from "@hypit/protocol";
import { canonicalize } from "@hypit/protocol";
-import {
- locateSpeechTiming,
-} from "./locate.js";
+import { alignSemanticTake } from "./local.js";
import { speechAlignmentProducers } from "./manifest.js";
function inline<T>(value: StoredValue | undefined, subject: string): T {
@@ -17,20 +15,23 @@ function inline<T>(value: StoredValue | undefined, subject: string): T {
/** Provider-neutral deterministic alignment; acoustic measurement remains an explicit upstream Need. */
export const speechAlignmentComponent = {
- producers: [{
- producer: speechAlignmentProducers.locate,
- handler: ({ inputs }) => ({
- outputs: {
- map: {
- kind: "inline",
- value: canonicalize(locateSpeechTiming(
- inline<Narrative>(inputs.narrative?.value, "Narrative"),
- inline<SpeechAudioBasis>(inputs.audio?.value, "SpeechAudioBasis"),
- inline<AlignedTranscriptEvidence>(inputs.evidence?.value, "AlignedTranscriptEvidence"),
- )),
+ producers: [
+ {
+ producer: speechAlignmentProducers.alignTake,
+ handler: ({ inputs }) => ({
+ outputs: {
+ take: {
+ kind: "inline",
+ value: canonicalize(alignSemanticTake(
+ inline<Narrative>(inputs.narrative?.value, "Narrative"),
+ inline<NarrativeExcerpt>(inputs.segment?.value, "NarrativeExcerpt"),
+ inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
+ inline<AlignedTranscriptEvidence>(inputs.evidence?.value, "AlignedTranscriptEvidence"),
+ )),
+ },
},
- },
- needs: {},
- }),
- }],
+ needs: {},
+ }),
+ },
+ ],
} satisfies ComponentPackage;
diff --git a/packages/speech-alignment/src/index.ts b/packages/speech-alignment/src/index.ts
index 53353906..b78d9d2f 100644
--- a/packages/speech-alignment/src/index.ts
+++ b/packages/speech-alignment/src/index.ts
@@ -1,7 +1,7 @@
export { alignWordGroups } from "./align.js";
export { speechAlignmentComponent } from "./component.js";
export { SpeechAlignmentError } from "./error.js";
-export { locateSpeechTiming } from "./locate.js";
+export { alignSemanticTake } from "./local.js";
export { speechAlignmentManifest, speechAlignmentModuleRef, speechAlignmentProducers } from "./manifest.js";
export { alignCharacters, alignmentCharacters, normalizeForAlignment } from "./normalize.js";
export type * from "./types.js";
diff --git a/packages/speech-alignment/src/local.ts b/packages/speech-alignment/src/local.ts
new file mode 100644
index 00000000..d8dd8bc0
--- /dev/null
+++ b/packages/speech-alignment/src/local.ts
@@ -0,0 +1,62 @@
+import type { SynchronizedMedia } from "@hypit/media";
+import type { Narrative, NarrativeExcerpt } from "@hypit/narrative";
+import { sealProgramSpace } from "@hypit/program-space";
+import type { SemanticTake } from "@hypit/speech";
+import type { AlignedTranscriptEvidence } from "@hypit/speech-evidence";
+
+import { locateAlignedSegmentTiming } from "./locate.js";
+import type { AlignmentBasis } from "./locate.js";
+import { materializeSemanticTake } from "./materialize.js";
+
+function segmentNarrative(narrative: Narrative, excerpt: NarrativeExcerpt): Narrative {
+ if (excerpt.kind !== "segment") throw new Error("SemanticTake alignment requires a Segment excerpt.");
+ const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id);
+ if (segment === undefined) throw new Error(`Narrative does not contain Segment ${excerpt.id}.`);
+ if (excerpt.tokenStart !== segment.tokenStart || excerpt.tokenEndExclusive !== segment.tokenEndExclusive) {
+ throw new Error(`NarrativeExcerpt ${excerpt.id} does not describe its authored Segment.`);
+ }
+ const tokens = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);
+ return {
+ segments: [{ ...segment, tokenStart: 0, tokenEndExclusive: tokens.length }],
+ tokens,
+ turns: narrative.turns
+ .filter((turn) => turn.segmentId === segment.id)
+ .map((turn) => ({
+ ...turn,
+ tokenStart: turn.tokenStart - segment.tokenStart,
+ tokenEndExclusive: turn.tokenEndExclusive - segment.tokenStart,
+ })),
+ selections: [],
+ moments: [],
+ semanticIndex: {
+ anchors: narrative.semanticIndex.anchors.filter((anchor) => anchor.segmentId === segment.id),
+ },
+ };
+}
+
+/** Align one normalized Take independently, then materialize its local semantic product. */
+export function alignSemanticTake(
+ narrative: Narrative,
+ excerpt: NarrativeExcerpt,
+ media: SynchronizedMedia,
+ evidence: AlignedTranscriptEvidence,
+): SemanticTake {
+ if (media.audio === undefined) throw new Error(`SemanticTake ${excerpt.id} has no normalized audio.`);
+ const localNarrative = segmentNarrative(narrative, excerpt);
+ const programSpace = sealProgramSpace({
+ durationSec: media.timeline.frameCount * media.timeline.frameRate.denominator
+ / media.timeline.frameRate.numerator,
+ frameRate: media.timeline.frameRate,
+ });
+ const basis: AlignmentBasis = {
+ programSpace,
+ audio: media.audio.artifact,
+ segments: [{ segmentId: excerpt.id, startFrame: 0, endFrameExclusive: media.timeline.frameCount }],
+ };
+ const map = locateAlignedSegmentTiming(localNarrative, basis, evidence);
+ return materializeSemanticTake(localNarrative, {
+ ...excerpt,
+ tokenStart: 0,
+ tokenEndExclusive: localNarrative.tokens.length,
+ }, media, map);
+}
diff --git a/packages/speech-alignment/src/locate.ts b/packages/speech-alignment/src/locate.ts
index 77f209ef..4311b35b 100644
--- a/packages/speech-alignment/src/locate.ts
+++ b/packages/speech-alignment/src/locate.ts
@@ -1,7 +1,8 @@
import { isDigest } from "@hypit/protocol";
+import type { BlobRef } from "@hypit/protocol";
import type { Narrative, NarrativeToken } from "@hypit/narrative";
import { programFrameSampleBoundary, programSpaceFrameCount } from "@hypit/program-space";
-import type { SpeechAudioBasis } from "@hypit/speech";
+import type { ProgramSpace } from "@hypit/program-space";
import type {
AlignedTranscriptEvidence,
SpeechActivitySpan,
@@ -16,6 +17,17 @@ import { SpeechAlignmentError } from "./error.js";
import { alignCharacters, alignmentCharacters } from "./normalize.js";
import type { AlignmentGroup, TimedSpeechSegment } from "./types.js";
+/** Package-private clock used while aligning exactly one normalized Segment Take. */
+export type AlignmentBasis = {
+ readonly programSpace: ProgramSpace;
+ readonly audio: BlobRef;
+ readonly segments: readonly [{
+ readonly segmentId: string;
+ readonly startFrame: 0;
+ readonly endFrameExclusive: number;
+ }];
+};
+
type MutableTiming = {
startSample: number;
endSampleExclusive: number;
@@ -63,7 +75,7 @@ function validateSampleWindow(
}
}
-function evidenceSampleFrames(basis: SpeechAudioBasis): number {
+function evidenceSampleFrames(basis: AlignmentBasis): number {
return programFrameSampleBoundary(
basis.programSpace,
programSpaceFrameCount(basis.programSpace),
@@ -71,7 +83,7 @@ function evidenceSampleFrames(basis: SpeechAudioBasis): number {
);
}
-function validateBasis(narrative: Narrative, basis: SpeechAudioBasis): void {
+function validateBasis(narrative: Narrative, basis: AlignmentBasis): void {
const { numerator, denominator } = basis.programSpace.frameRate;
if (!Number.isSafeInteger(numerator) || numerator <= 0
|| !Number.isSafeInteger(denominator) || denominator <= 0) {
@@ -82,28 +94,28 @@ function validateBasis(narrative: Narrative, basis: SpeechAudioBasis): void {
|| basis.audio.mediaType !== "audio/wav" || !Number.isSafeInteger(basis.audio.size) || basis.audio.size < 0) {
fail("SPEECH_AUDIO_DIGEST", "SpeechBasis audio BlobRef is invalid.");
}
- if (basis.segments.length !== narrative.segments.length) {
- fail("SPEECH_BASIS_SEGMENTS", "SpeechAudioBasis must cover every Narrative Segment exactly once.");
+ if (narrative.segments.length !== 1 || basis.segments.length !== 1) {
+ fail("SPEECH_BASIS_SEGMENTS", "Speech alignment accepts exactly one normalized Segment Take.");
}
let previousEnd = 0;
for (const [index, segment] of basis.segments.entries()) {
const expected = narrative.segments[index]!;
if (segment.segmentId !== expected.id) {
- fail("SPEECH_BASIS_SEGMENTS", `SpeechAudioBasis Segment ${segment.segmentId} does not match ${expected.id}.`);
+ fail("SPEECH_BASIS_SEGMENTS", `Alignment Segment ${segment.segmentId} does not match ${expected.id}.`);
}
if (!Number.isSafeInteger(segment.startFrame) || !Number.isSafeInteger(segment.endFrameExclusive)
|| segment.startFrame !== previousEnd || segment.endFrameExclusive <= segment.startFrame
|| segment.endFrameExclusive > frameCount) {
- fail("SPEECH_BASIS_SEGMENTS", `SpeechAudioBasis Segment ${segment.segmentId} has an invalid frame window.`);
+ fail("SPEECH_BASIS_SEGMENTS", `Alignment Segment ${segment.segmentId} has an invalid frame window.`);
}
previousEnd = segment.endFrameExclusive;
}
if (previousEnd !== frameCount) {
- fail("SPEECH_BASIS_SEGMENTS", "SpeechAudioBasis Segments must cover ProgramSpace exactly.");
+ fail("SPEECH_BASIS_SEGMENTS", "Alignment Segments must cover ProgramSpace exactly.");
}
}
-function validateEvidence(basis: SpeechAudioBasis, evidence: AlignedTranscriptEvidence): void {
+function validateEvidence(basis: AlignmentBasis, evidence: AlignedTranscriptEvidence): void {
const limit = evidenceSampleFrames(basis);
for (const [passageIndex, passage] of evidence.passages.entries()) {
validateSampleWindow(passage, limit, `Passage ${passageIndex + 1}`);
@@ -134,7 +146,7 @@ function sampleMidpoint(value: {
/** Assign acoustic evidence to authored Segments only where both clocks are explicitly connected. */
function partitionEvidence(
- basis: SpeechAudioBasis,
+ basis: AlignmentBasis,
evidence: AlignedTranscriptEvidence,
): readonly SegmentEvidence[] {
const ranges = basis.segments.map((segment) => ({
@@ -314,9 +326,9 @@ function locatePairedGroup(
}
function speechBounds(
- basis: SpeechAudioBasis,
+ basis: AlignmentBasis,
segment: SegmentEvidence,
- basisSegment: SpeechAudioBasis["segments"][number],
+ basisSegment: AlignmentBasis["segments"][number],
): { readonly start: number; readonly end: number } {
const spans = segment.speechActivity;
return spans.length
@@ -361,7 +373,7 @@ function fillMissingTiming(
return values as MutableTiming[];
}
-function frameForEvidenceSample(basis: SpeechAudioBasis, sample: number): number {
+function frameForEvidenceSample(basis: AlignmentBasis, sample: number): number {
const numerator = BigInt(sample) * BigInt(basis.programSpace.frameRate.numerator);
const denominator = 16_000n * BigInt(basis.programSpace.frameRate.denominator);
const frame = (numerator * 2n + denominator) / (denominator * 2n);
@@ -370,9 +382,9 @@ function frameForEvidenceSample(basis: SpeechAudioBasis, sample: number): number
}
/** Locate every Script token from one provider-neutral acoustic evidence pass. */
-export function locateSpeechTiming(
+export function locateAlignedSegmentTiming(
narrative: Narrative,
- basis: SpeechAudioBasis,
+ basis: AlignmentBasis,
evidence: AlignedTranscriptEvidence,
): CompleteSemanticMap {
validateBasis(narrative, basis);
diff --git a/packages/speech-alignment/src/manifest.ts b/packages/speech-alignment/src/manifest.ts
index 48005f6d..c0cb6ce7 100644
--- a/packages/speech-alignment/src/manifest.ts
+++ b/packages/speech-alignment/src/manifest.ts
@@ -1,12 +1,12 @@
+import { mediaDependency, mediaTypes } from "@hypit/media";
import { narrativeDependency, narrativeTypes } from "@hypit/narrative";
import { speechDependency, speechTypes } from "@hypit/speech";
import { speechEvidenceDependency, speechEvidenceTypes } from "@hypit/speech-evidence";
-import { semanticMapDependency, semanticMapTypes } from "@hypit/semantic-map";
import type { ModuleManifest, ProducerRef } from "@hypit/protocol";
export const speechAlignmentModuleRef = { name: "@hypit/speech-alignment", version: "1" } as const;
export const speechAlignmentProducers = {
- locate: { module: speechAlignmentModuleRef, name: "locate-speech" },
+ alignTake: { module: speechAlignmentModuleRef, name: "align-semantic-take" },
} satisfies Record<string, ProducerRef>;
export const speechAlignmentManifest: ModuleManifest = {
@@ -14,21 +14,24 @@ export const speechAlignmentManifest: ModuleManifest = {
name: speechAlignmentModuleRef.name,
version: speechAlignmentModuleRef.version,
dependencies: [
+ mediaDependency,
narrativeDependency,
speechDependency,
speechEvidenceDependency,
- semanticMapDependency,
],
types: [],
capabilities: [],
- producers: [{
- name: speechAlignmentProducers.locate.name,
- inputs: [
- { name: "narrative", type: narrativeTypes.narrative },
- { name: "audio", type: speechTypes.audioBasis },
- { name: "evidence", type: speechEvidenceTypes.alignedTranscript },
- ],
- outputs: [{ name: "map", type: semanticMapTypes.complete }],
- needs: [],
- }],
+ producers: [
+ {
+ name: speechAlignmentProducers.alignTake.name,
+ inputs: [
+ { name: "narrative", type: narrativeTypes.narrative },
+ { name: "segment", type: narrativeTypes.excerpt },
+ { name: "media", type: mediaTypes.synchronized },
+ { name: "evidence", type: speechEvidenceTypes.alignedTranscript },
+ ],
+ outputs: [{ name: "take", type: speechTypes.semanticTake }],
+ needs: [],
+ },
+ ],
};
diff --git a/packages/speech-alignment/src/materialize.ts b/packages/speech-alignment/src/materialize.ts
new file mode 100644
index 00000000..1249a602
--- /dev/null
+++ b/packages/speech-alignment/src/materialize.ts
@@ -0,0 +1,76 @@
+import type { Narrative, NarrativeExcerpt } from "@hypit/narrative";
+import type { SynchronizedMedia } from "@hypit/media";
+import type { CompleteSemanticMap } from "@hypit/semantic-map";
+import type { SemanticTake } from "@hypit/speech";
+import { assertSemanticTakeIdentity, sealSemanticTake } from "@hypit/speech";
+
+function authoredSegment(narrative: Narrative, excerpt: NarrativeExcerpt): Narrative["segments"][number] {
+ if (excerpt.kind !== "segment") throw new Error("SemanticTake materialization requires a Segment excerpt.");
+ const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id);
+ if (segment === undefined) throw new Error(`Narrative does not contain Segment ${excerpt.id}.`);
+ if (excerpt.tokenStart !== segment.tokenStart || excerpt.tokenEndExclusive !== segment.tokenEndExclusive) {
+ throw new Error(`NarrativeExcerpt ${excerpt.id} does not describe its authored Segment.`);
+ }
+ return segment;
+}
+
+function localFrame(frame: number, frameCount: number, label: string): number {
+ if (!Number.isSafeInteger(frame) || frame < 0 || frame > frameCount) {
+ throw new Error(`${label} lies outside its normalized Segment Take.`);
+ }
+ return frame;
+}
+
+/**
+ * Copy one Segment-local alignment into a self-contained semantic Take.
+ */
+export function materializeSemanticTake(
+ narrative: Narrative,
+ excerpt: NarrativeExcerpt,
+ media: SynchronizedMedia,
+ map: CompleteSemanticMap,
+): SemanticTake {
+ const segment = authoredSegment(narrative, excerpt);
+ const localFrameCount = media.timeline.frameCount;
+ const timedById = new Map(map.tokens.map((token) => [token.tokenId, token]));
+ const tokens = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive).map((token) => {
+ const timed = timedById.get(token.id);
+ if (timed === undefined || timed.segmentId !== segment.id) {
+ throw new Error(`Semantic map does not locate authored Token ${token.id}.`);
+ }
+ return {
+ tokenId: token.id,
+ segmentId: segment.id,
+ text: token.text,
+ startAnchorId: token.startAnchorId,
+ endAnchorId: token.endAnchorId,
+ startFrame: localFrame(timed.startFrame, localFrameCount, `Token ${token.id}`),
+ endFrameExclusive: localFrame(timed.endFrameExclusive, localFrameCount, `Token ${token.id}`),
+ };
+ });
+ const anchorsById = new Map(map.anchors.map((anchor) => [anchor.identity, anchor]));
+ const anchors = narrative.semanticIndex.anchors
+ .filter((anchor) => anchor.segmentId === segment.id)
+ .map((anchor) => {
+ const measured = anchorsById.get(anchor.id);
+ if (measured === undefined) throw new Error(`Semantic map does not locate authored Anchor ${anchor.id}.`);
+ return {
+ identity: anchor.id,
+ frame: localFrame(measured.frame, localFrameCount, `Anchor ${anchor.id}`),
+ };
+ });
+ const take = sealSemanticTake({
+ media,
+ segment: {
+ segmentId: segment.id,
+ startAnchorId: segment.startAnchorId,
+ endAnchorId: segment.endAnchorId,
+ startFrame: 0,
+ endFrameExclusive: media.timeline.frameCount,
+ },
+ tokens,
+ anchors,
+ });
+ assertSemanticTakeIdentity(take);
+ return take;
+}
diff --git a/packages/speech-alignment/src/types.ts b/packages/speech-alignment/src/types.ts
index 9139cdf3..d81c11e0 100644
--- a/packages/speech-alignment/src/types.ts
+++ b/packages/speech-alignment/src/types.ts
@@ -5,13 +5,6 @@ export type {
SpeechTranscriptPassage,
SpeechWordEvidence,
} from "@hypit/speech-evidence";
-export type {
- CompleteSemanticMap,
- SemanticTimePoint,
- TimedSpeechToken,
-} from "@hypit/semantic-map";
-export type { SpeechAudioBasis } from "@hypit/speech";
-
/** Private vocabulary of the alignment implementation, not part of SemanticMap. */
export type AlignmentRelation =
| "exact"
diff --git a/packages/speech-alignment/test/speech-align.test.ts b/packages/speech-alignment/test/speech-align.test.ts
index 9ee857b0..001ace81 100644
--- a/packages/speech-alignment/test/speech-align.test.ts
+++ b/packages/speech-alignment/test/speech-align.test.ts
@@ -1,8 +1,6 @@
import type { Narrative } from "@hypit/narrative";
import { sealProgramSpace } from "@hypit/program-space";
-import type { ProgramSpace } from "@hypit/program-space";
-import { sealSpeechBasis } from "@hypit/speech";
-import type { SpeechAudioBasis, SpeechBasis } from "@hypit/speech";
+import { materializeSemanticTake } from "../src/materialize.js";
import { sealAlignedTranscriptEvidence } from "@hypit/speech-evidence";
import type { AlignedTranscriptEvidence, SpeechCharacterEvidence, SpeechWordEvidence } from "@hypit/speech-evidence";
import assert from "node:assert/strict";
@@ -13,8 +11,9 @@ import { parseScript } from "@hypit/script";
import {
SpeechAlignmentError,
alignWordGroups,
- locateSpeechTiming,
} from "@hypit/speech-alignment";
+import { locateAlignedSegmentTiming } from "../src/locate.js";
+import type { AlignmentBasis } from "../src/locate.js";
type WordFixture = {
readonly text: string;
@@ -49,7 +48,7 @@ function relations(
}
function evidence(args: {
- readonly basis: SpeechAudioBasis;
+ readonly basis: AlignmentBasis;
readonly durationSec?: number;
readonly startSec?: number;
readonly endSec?: number;
@@ -76,32 +75,21 @@ function evidence(args: {
function speechBasis(
narrative: Narrative,
durationSec = 2,
- windows?: readonly { readonly startSec: number; readonly endSec: number }[],
-): SpeechAudioBasis {
+): AlignmentBasis {
+ if (narrative.segments.length !== 1) throw new Error("Test alignment requires one Segment.");
const programSpace = sealProgramSpace({
durationSec,
frameRate: { numerator: 1_000, denominator: 1 },
});
const audioDigest = fixtureDigest(`fixture:audio:${narrative.segments.map((segment) => segment.id).join("+")}:${durationSec}`);
- const segments = narrative.segments.map((segment, index) => ({
- segmentId: segment.id,
- startFrame: Math.round((windows?.[index]?.startSec ?? durationSec * index / narrative.segments.length) * 1_000),
- endFrameExclusive: Math.round((windows?.[index]?.endSec ?? durationSec * (index + 1) / narrative.segments.length) * 1_000),
- }));
- const take = sealSpeechBasis({
+ return {
programSpace,
audio: { kind: "blob", digest: audioDigest, size: 1, mediaType: "audio/wav" },
- visualTrack: { clips: [] },
- segments,
- });
- return audioProjection(take);
-}
-
-function audioProjection(basis: SpeechBasis): SpeechAudioBasis {
- return {
- programSpace: basis.programSpace,
- audio: basis.audio,
- segments: basis.segments,
+ segments: [{
+ segmentId: narrative.segments[0]!.id,
+ startFrame: 0,
+ endFrameExclusive: Math.round(durationSec * 1_000),
+ }],
};
}
@@ -110,11 +98,8 @@ function locate(
args: Omit<Parameters<typeof evidence>[0], "basis">,
) {
const durationSec = args.durationSec ?? 2;
- const basis = speechBasis(narrative, durationSec, [{
- startSec: args.startSec ?? 0,
- endSec: args.endSec ?? durationSec,
- }]);
- return locateSpeechTiming(narrative, basis, evidence({ ...args, basis }));
+ const basis = speechBasis(narrative, durationSec);
+ return locateAlignedSegmentTiming(narrative, basis, evidence({ ...args, basis }));
}
function characters(
@@ -153,6 +138,34 @@ test("exact transcript words cover every Script and Segment anchor", () => {
assert.equal(new Set(map.anchors.map((anchor) => anchor.identity)).size, map.anchors.length);
});
+test("a measured Segment-local map becomes a self-contained SemanticTake", () => {
+ const narrative = parseScript("materialize.svml", "<line>Hello world.</line>");
+ const basis = speechBasis(narrative, 2);
+ const map = locate(narrative, {
+ words: [
+ { text: "Hello", startSec: 0.1, endSec: 0.4 },
+ { text: "world", startSec: 0.5, endSec: 0.9 },
+ ],
+ });
+ const segment = narrative.segments[0]!;
+ const take = materializeSemanticTake(
+ narrative,
+ { kind: "segment", id: segment.id, tokenStart: segment.tokenStart, tokenEndExclusive: segment.tokenEndExclusive },
+ {
+ timeline: { frameRate: { numerator: 1_000, denominator: 1 }, frameCount: 2_000 },
+ visual: { artifact: { kind: "blob", digest: fixtureDigest("materialize:video"), size: 1, mediaType: "video/mp4" }, width: 720, height: 1280 },
+ audio: { artifact: { kind: "blob", digest: fixtureDigest("materialize:audio"), size: 1, mediaType: "audio/wav" } },
+ },
+ map,
+ );
+ assert.deepEqual(take.tokens.map((token) => [token.text, token.startFrame, token.endFrameExclusive]), [
+ ["Hello", 100, 400],
+ ["world", 500, 900],
+ ]);
+ assert.equal(take.segment.startFrame, 0);
+ assert.equal(take.segment.endFrameExclusive, 2_000);
+});
+
test("M:1 uses evidence character times instead of dividing a merged word by length", () => {
const narrative = parseScript("merge.svml", "<line>can not</line>");
const merged = [{ text: "cannot", startSec: 0.1, endSec: 0.78, score: 0.93 }];
@@ -258,63 +271,6 @@ test("VAD bounds contain missing tokens when a Script Segment has no recognized
);
});
-test("multiple Script Segments stay independent even when evidence records arrive out of order", () => {
- const narrative = parseScript("segments.svml", "<one>Hello.</one><two>Goodbye.</two>");
- const basis = speechBasis(narrative, 2, [
- { startSec: 0, endSec: 1 },
- { startSec: 1, endSec: 2 },
- ]);
- const map = locateSpeechTiming(narrative, basis, sealAlignedTranscriptEvidence({
- passages: [
- {
- words: [wordEvidence({ text: "Goodbye", startSec: 1.2, endSec: 1.6 })],
- chars: [],
- },
- {
- words: [wordEvidence({ text: "Hello", startSec: 0.2, endSec: 0.55 })],
- chars: [],
- },
- ],
- }));
-
- assert.deepEqual(map.tokens.map((token) => [token.segmentId, token.startFrame, token.endFrameExclusive]), [
- ["one", 200, 550],
- ["two", 1_200, 1_600],
- ]);
-});
-
-test("Segment anchors preserve exact frame cuts without a seconds round trip", () => {
- const narrative = parseScript("frame-cuts.svml", "<one>Alpha.</one><two>Beta.</two>");
- const take = sealSpeechBasis({
- programSpace: sealProgramSpace({
- durationSec: 1,
- frameRate: { numerator: 24, denominator: 1 },
- }),
- audio: { kind: "blob", digest: fixtureDigest("frame-cuts:audio"), size: 1, mediaType: "audio/wav" },
- visualTrack: { clips: [] },
- segments: [
- { segmentId: "one", startFrame: 0, endFrameExclusive: 7 },
- { segmentId: "two", startFrame: 7, endFrameExclusive: 24 },
- ],
- });
- const map = locateSpeechTiming(narrative, audioProjection(take), sealAlignedTranscriptEvidence({
- passages: [{
- startSample: 0,
- endSampleExclusive: 16_000,
- words: [
- { text: "Alpha", startSample: 1_000, endSampleExclusive: 3_000 },
- { text: "Beta", startSample: 6_000, endSampleExclusive: 10_000 },
- ],
- chars: [],
- }],
- }));
- const frameByAnchor = new Map(map.anchors.map((anchor) => [anchor.identity, anchor.frame]));
- const structuralFrames = narrative.semanticIndex.anchors
- .filter((anchor) => anchor.kind === "segment-start" || anchor.kind === "segment-end")
- .map((anchor) => frameByAnchor.get(anchor.id));
- assert.deepEqual(structuralFrames, [0, 7, 7, 24]);
-});
-
test("overlapping evidence word windows reach the map overlapping", () => {
// Two spoken words whose measured windows overlap. Nothing here knows whether
// that is a real overlap or a wobble, and no consumer of the map is bound to
@@ -346,7 +302,7 @@ test("locating is total: every Script token carries a window", () => {
assert.equal(diverged.tokens.every((token) => Number.isSafeInteger(token.startFrame)), true);
});
-test("Evidence is interpreted only through the explicitly connected SpeechAudioBasis", () => {
+test("Evidence is interpreted only through the explicitly connected alignment clock", () => {
const narrative = parseScript("affinity.svml", "<line>Hello world.</line>");
const basis = speechBasis(narrative, 2);
const mismatched = evidence({
@@ -357,12 +313,12 @@ test("Evidence is interpreted only through the explicitly connected SpeechAudioB
durationSec: 2,
frameRate: { numerator: 30, denominator: 1 },
});
- const anotherBasis = {
+ const anotherBasis: AlignmentBasis = {
...basis,
programSpace: anotherSpace,
segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 60 }],
};
- const map = locateSpeechTiming(narrative, anotherBasis, mismatched);
+ const map = locateAlignedSegmentTiming(narrative, anotherBasis, mismatched);
assert.equal(map.tokens[0]?.startFrame, 3);
});
@@ -373,12 +329,12 @@ test("the final map is quantized once into the selected ProgramSpace", () => {
durationSec: 1,
frameRate: { numerator: 30, denominator: 1 },
});
- const basis: SpeechAudioBasis = {
+ const basis: AlignmentBasis = {
...original,
programSpace,
segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 30 }],
};
- const map = locateSpeechTiming(narrative, basis, evidence({
+ const map = locateAlignedSegmentTiming(narrative, basis, evidence({
basis,
durationSec: 1,
endSec: 1,
diff --git a/packages/speech-basis/README.md b/packages/speech-basis/README.md
index 2af470e7..3b870678 100644
--- a/packages/speech-basis/README.md
+++ b/packages/speech-basis/README.md
@@ -4,15 +4,15 @@ Reference Product/Projection package for an atomic speech take.
`SpeechBasis` is the indivisible Product that binds audio, visual clips and ProgramSpace. Its
identity belongs to the enclosing Core Record; the domain value does not repeat a self digest or
-copy an upstream identity. Four ordinary deterministic Producers project it into `ProgramSpace`,
-`SpeechAudioBasis`, `VisualTrack` and `AudioTrack`. Core has no Projection primitive and never treats
+copy an upstream identity. Three ordinary deterministic Producers project it into `ProgramSpace`,
+`VisualTrack` and `AudioTrack`. Core has no Projection primitive and never treats
these outputs as ports of one mutable workflow node.
This gives the author several independent Logical Outputs while preserving one shared generation
Operation. Selecting an Existing `SpeechBasis` stops generation and leaves both Projections usable;
selecting only an Existing visual Candidate does not change the independent audio path.
-`speechBasisComponent` enumerates all four Producer facets. Their identities are checked against the
+`speechBasisComponent` enumerates all three Producer facets. Their identities are checked against the
static `speechBasisManifest` and can be activated by any compatible compute Host selected by Source
imports without a Driver or Runtime source change. The component has no Provider,
Artifact access, queue or credentials.
diff --git a/packages/speech-basis/package.json b/packages/speech-basis/package.json
index 43033a05..29c32c48 100644
--- a/packages/speech-basis/package.json
+++ b/packages/speech-basis/package.json
@@ -23,6 +23,7 @@
"@hypit/core": "workspace:*",
"@hypit/driver-node": "workspace:*",
"@hypit/media-pipeline": "workspace:*",
+ "@hypit/svs": "workspace:*",
"@hypit/validation": "workspace:*"
}
}
diff --git a/packages/speech-basis/src/component.ts b/packages/speech-basis/src/component.ts
index 5f5b0cdd..ae3e9257 100644
--- a/packages/speech-basis/src/component.ts
+++ b/packages/speech-basis/src/component.ts
@@ -6,7 +6,7 @@ import { canonicalize } from "@hypit/protocol";
import {
speechBasisProducers,
} from "./manifest.js";
-import { projectSpeechAudio, projectSpeechAudioTrack, projectSpeechProgramSpace, projectSpeechVisual } from "./projection.js";
+import { projectSpeechAudioTrack, projectSpeechProgramSpace, projectSpeechVisual } from "./projection.js";
function speechBasis(value: StoredValue | undefined): SpeechBasis {
if (value?.kind !== "inline") throw new Error("SpeechBasis must be inline");
@@ -28,18 +28,6 @@ export const speechBasisComponent = {
needs: {},
}),
},
- {
- producer: speechBasisProducers.projectAudio,
- handler: ({ inputs }) => ({
- outputs: {
- audio: {
- kind: "inline",
- value: canonicalize(projectSpeechAudio(speechBasis(inputs.basis?.value))),
- },
- },
- needs: {},
- }),
- },
{
producer: speechBasisProducers.projectVisual,
handler: ({ inputs }) => ({
diff --git a/packages/speech-basis/src/fragment.ts b/packages/speech-basis/src/fragment.ts
index 9e35cb33..e26ed170 100644
--- a/packages/speech-basis/src/fragment.ts
+++ b/packages/speech-basis/src/fragment.ts
@@ -19,12 +19,6 @@ export const speechBasisProjectionFragment = sealGraphFragment({
inputs: { basis: input("basis") },
result: { kind: "output", name: "programSpace" },
},
- {
- id: "project-audio",
- producer: speechBasisProducers.projectAudio,
- inputs: { basis: input("basis") },
- result: { kind: "output", name: "audio" },
- },
{
id: "project-visual",
producer: speechBasisProducers.projectVisual,
@@ -44,11 +38,6 @@ export const speechBasisProjectionFragment = sealGraphFragment({
type: programSpaceTypes.programSpace,
root: operation("project-program-space"),
},
- {
- name: "audio",
- type: speechTypes.audioBasis,
- root: operation("project-audio"),
- },
{
name: "visual",
type: compositionTypes.visualTrack,
diff --git a/packages/speech-basis/src/index.ts b/packages/speech-basis/src/index.ts
index d249b951..002ee9b4 100644
--- a/packages/speech-basis/src/index.ts
+++ b/packages/speech-basis/src/index.ts
@@ -5,4 +5,4 @@ export {
speechBasisModuleRef,
speechBasisProducers,
} from "./manifest.js";
-export { projectSpeechAudio, projectSpeechAudioTrack, projectSpeechProgramSpace, projectSpeechVisual } from "./projection.js";
+export { projectSpeechAudioTrack, projectSpeechProgramSpace, projectSpeechVisual } from "./projection.js";
diff --git a/packages/speech-basis/src/manifest.ts b/packages/speech-basis/src/manifest.ts
index bfb89afd..b1bb2f1d 100644
--- a/packages/speech-basis/src/manifest.ts
+++ b/packages/speech-basis/src/manifest.ts
@@ -6,7 +6,6 @@ import type { ModuleManifest, ProducerRef } from "@hypit/protocol";
export const speechBasisModuleRef = { name: "@hypit/speech-basis", version: "1" } as const;
export const speechBasisProducers = {
- projectAudio: { module: speechBasisModuleRef, name: "project-audio" },
projectAudioTrack: { module: speechBasisModuleRef, name: "project-audio-track" },
projectVisual: { module: speechBasisModuleRef, name: "project-visual" },
projectProgramSpace: { module: speechBasisModuleRef, name: "project-program-space" },
@@ -30,12 +29,6 @@ export const speechBasisManifest: ModuleManifest = {
outputs: [{ name: "programSpace", type: programSpaceTypes.programSpace }],
needs: [],
},
- {
- name: speechBasisProducers.projectAudio.name,
- inputs: [{ name: "basis", type: speechTypes.basis }],
- outputs: [{ name: "audio", type: speechTypes.audioBasis }],
- needs: [],
- },
{
name: speechBasisProducers.projectVisual.name,
inputs: [{ name: "basis", type: speechTypes.basis }],
diff --git a/packages/speech-basis/src/projection.ts b/packages/speech-basis/src/projection.ts
index e7dcc15d..48047e46 100644
--- a/packages/speech-basis/src/projection.ts
+++ b/packages/speech-basis/src/projection.ts
@@ -1,7 +1,7 @@
import { programSpaceSampleFrames } from "@hypit/program-space";
import type { ProgramSpace } from "@hypit/program-space";
import { assertSpeechBasisIdentity } from "@hypit/speech";
-import type { SpeechAudioBasis, SpeechBasis } from "@hypit/speech";
+import type { SpeechBasis } from "@hypit/speech";
import { lowerRestrictedSpeechVisualPresents } from "@hypit/media-track";
import { sealAudioTrack, sealVisualTrack } from "@hypit/composition";
import type { AudioTrack, VisualTrack } from "@hypit/composition";
@@ -11,15 +11,6 @@ export function projectSpeechProgramSpace(basis: SpeechBasis): ProgramSpace {
return basis.programSpace;
}
-export function projectSpeechAudio(basis: SpeechBasis): SpeechAudioBasis {
- assertSpeechBasisIdentity(basis);
- return {
- programSpace: basis.programSpace,
- audio: basis.audio,
- segments: basis.segments,
- };
-}
-
export function projectSpeechVisual(basis: SpeechBasis): VisualTrack {
assertSpeechBasisIdentity(basis);
const trackId = `speech-visual:${basis.segments.map((segment) => segment.segmentId).join("+")}`;
diff --git a/packages/speech-basis/test/product.test.ts b/packages/speech-basis/test/product.test.ts
index e968134e..eb3afbb0 100644
--- a/packages/speech-basis/test/product.test.ts
+++ b/packages/speech-basis/test/product.test.ts
@@ -30,7 +30,6 @@ import type {
TypeRef,
} from "@hypit/protocol";
import {
- projectSpeechAudio,
projectSpeechAudioTrack,
projectSpeechProgramSpace,
projectSpeechVisual,
@@ -40,6 +39,7 @@ import {
} from "@hypit/speech-basis";
import { TypeValidatorRegistry } from "@hypit/validation";
import { mediaTrackManifest } from "@hypit/media-track";
+import { svsManifest } from "@hypit/svs";
const testModule = { name: "example.speech-basis-product", version: "0.0.0" } as const;
const requestType = { module: testModule, name: "SpeechRequest" } satisfies TypeRef;
@@ -64,6 +64,7 @@ const closure = createResolvedClosure([
...videoContractManifests,
mediaPipelineManifest,
mediaTrackManifest,
+ svsManifest,
speechBasisManifest,
testManifest,
]);
@@ -143,7 +144,7 @@ function createGraph(program: LinkedProgram): CompiledGraph {
},
{
id: "opening.audio",
- type: speechTypes.audioBasis,
+ type: compositionTypes.audioTrack,
primary: "opening.audio.project",
},
{
@@ -171,7 +172,7 @@ function createGraph(program: LinkedProgram): CompiledGraph {
},
{
id: "opening.audio.project",
- type: speechTypes.audioBasis,
+ type: compositionTypes.audioTrack,
root: { kind: "operation", result: operation("project-opening-audio") },
},
{
@@ -200,9 +201,9 @@ function createGraph(program: LinkedProgram): CompiledGraph {
},
{
id: "project-opening-audio",
- producer: speechBasisProducers.projectAudio,
+ producer: speechBasisProducers.projectAudioTrack,
inputs: { basis: output("opening.take") },
- result: { kind: "output", name: "audio", record: "audio:opening" },
+ result: { kind: "output", name: "track", record: "audio:opening" },
},
{
id: "project-opening-visual",
diff --git a/packages/speech-evidence/src/index.ts b/packages/speech-evidence/src/index.ts
index 929ff23a..47ee3c86 100644
--- a/packages/speech-evidence/src/index.ts
+++ b/packages/speech-evidence/src/index.ts
@@ -24,8 +24,8 @@ export type SpeechActivitySpan = {
/**
* One acoustic passage reported by the evidence provider. It deliberately has
- * no authored Segment identity; assigning evidence to Script Segments belongs
- * to Speech Alignment, where the connected SpeechAudioBasis is available.
+ * no authored Segment identity; assigning evidence to the explicitly connected
+ * Script Segment belongs to the later deterministic Speech Alignment step.
*/
export type SpeechTranscriptPassage = {
readonly startSample?: number;
diff --git a/packages/speech-spine/README.md b/packages/speech-spine/README.md
index 605962c0..7f7b8f04 100644
--- a/packages/speech-spine/README.md
+++ b/packages/speech-spine/README.md
@@ -6,19 +6,15 @@ This package contains no Core behavior, Provider credentials, queues, or API
clients. It only composes ordinary Producers from the leaf packages into
auditable reusable graph contributions:
-- atomic SpeechBasis audio/visual projections;
-- WhisperX evidence plus the provider-neutral speech SemanticMap;
+- atomic SpeechBasis audio/visual projections after assembly;
+- already-semantic Take assembly plus the provider-neutral speech SemanticMap;
- timed caption projection.
-The package also owns `<speech:Spine>`. The Spine explicitly declares its `frame-rate` and its
-restricted visual base (`visual-frame`, `visual-appearance`, `visual-z`); each
-`<speech:Take>` connects one `NarrativeExcerpt` and exactly one of:
-
-- `video={...}` — an ordinary generated video Blob. The Surface expands inspection, stream
- selection and A/V normalization before assembly;
-- `audio={...}` — an ordinary voice Blob. The Surface expands audio-authoritative normalization;
- this Take contributes no visual clip and never invents a black frame;
-- `media={...}` — an already prepared `SynchronizedMedia` value, connected directly.
+The package also owns `<speech:Spine>`. The Spine consumes the shared program `Clock` (with a
+legacy inline `frame-rate` accepted during migration) and its restricted visual base
+(`visual-frame`, `visual-appearance`, `visual-z`); each
+`<speech:Take>` connects one self-contained `SemanticTake`. Media normalization, acoustic evidence,
+and local semantic projection happen before the Spine is entered.
Visual Takes may override the Spine base with `frame`, `appearance` and `z`. That is the complete
visual authority of Speech Spine: one same-source visual clip may be placed and stacked. Motion,
@@ -26,6 +22,6 @@ transitions, sequences and independent pictures remain ordinary Media Tracks.
The Surface then expands an immutable append fold, one timeline-audio render and ordinary
SpeechBasis projections. Whitespace and child layout do not create ports, and Core receives no
-variadic node, media-normalization rule or Speech special case.
+variadic node, media-normalization rule, provider call, or global alignment special case.
The Elaborator expands them hygienically before SVML Core sees the graph.
diff --git a/packages/speech-spine/package.json b/packages/speech-spine/package.json
index a41e7e58..a5f0c843 100644
--- a/packages/speech-spine/package.json
+++ b/packages/speech-spine/package.json
@@ -17,11 +17,11 @@
"@hypit/elaborator": "workspace:*",
"@hypit/media": "workspace:*",
"@hypit/media-pipeline": "workspace:*",
- "@hypit/narrative": "workspace:*",
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
"@hypit/speech": "workspace:*",
"@hypit/speech-basis": "workspace:*",
+ "@hypit/semantic-map": "workspace:*",
"@hypit/spatial": "workspace:*",
"@hypit/svs": "workspace:*",
"@hypit/markup": "workspace:*"
diff --git a/packages/speech-spine/src/component.ts b/packages/speech-spine/src/component.ts
index 675d2e74..f5332902 100644
--- a/packages/speech-spine/src/component.ts
+++ b/packages/speech-spine/src/component.ts
@@ -1,12 +1,13 @@
import type { ComponentPackage } from "@hypit/component-kit";
-import type { NarrativeExcerpt } from "@hypit/narrative";
-import type { SynchronizedMedia, TimelineAudio } from "@hypit/media";
+import type { TimelineAudio } from "@hypit/media";
import type { StoredValue } from "@hypit/protocol";
import { canonicalize } from "@hypit/protocol";
+import type { SemanticTake } from "@hypit/speech";
import type { ContentFit, SpatialFrame } from "@hypit/spatial";
import { speechSpineProducers } from "./manifest.js";
import { appendSpeechSpineAudioTake, appendSpeechSpineVisualTake, assembleSpeechBasis, compileSpeechSpineAudio, createSpeechSpineSet } from "./program.js";
+import { assembleSemanticTakeMap } from "./semantic.js";
import type { SpeechSpineProgram, SpeechSpineSet, SpeechSpineVisualSpec } from "./types.js";
function inline<T>(value: StoredValue | undefined, subject: string): T {
@@ -29,8 +30,7 @@ export const speechSpineComponent = {
outputs: { set: { kind: "inline", value: canonicalize(appendSpeechSpineAudioTake(
inline<SpeechSpineSet>(inputs.set?.value, "SpeechSpineSet"),
inline<SpeechSpineProgram>(inputs.program?.value, "SpeechSpineProgram"),
- inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
- inline<NarrativeExcerpt>(inputs.segment?.value, "NarrativeExcerpt"),
+ inline<SemanticTake>(inputs.take?.value, "SemanticTake"),
)) } },
needs: {},
}),
@@ -41,8 +41,7 @@ export const speechSpineComponent = {
outputs: { set: { kind: "inline", value: canonicalize(appendSpeechSpineVisualTake(
inline<SpeechSpineSet>(inputs.set?.value, "SpeechSpineSet"),
inline<SpeechSpineProgram>(inputs.program?.value, "SpeechSpineProgram"),
- inline<SynchronizedMedia>(inputs.media?.value, "SynchronizedMedia"),
- inline<NarrativeExcerpt>(inputs.segment?.value, "NarrativeExcerpt"),
+ inline<SemanticTake>(inputs.take?.value, "SemanticTake"),
inline<SpatialFrame>(inputs.frame?.value, "SpatialFrame"),
inline<ContentFit>(inputs.fit?.value, "ContentFit"),
inline<SpeechSpineVisualSpec>(inputs.visualSpec?.value, "SpeechSpineVisualSpec"),
@@ -71,5 +70,19 @@ export const speechSpineComponent = {
needs: {},
}),
},
+ {
+ producer: speechSpineProducers.assembleSemanticMap,
+ handler: ({ inputs }) => ({
+ outputs: {
+ map: {
+ kind: "inline",
+ value: canonicalize(assembleSemanticTakeMap(
+ inline<SpeechSpineSet>(inputs.set?.value, "SpeechSpineSet").takes.map((take) => take.semantic),
+ )),
+ },
+ },
+ needs: {},
+ }),
+ },
],
} satisfies ComponentPackage;
diff --git a/packages/speech-spine/src/fragment.ts b/packages/speech-spine/src/fragment.ts
index 4b8cf0bd..32f4554b 100644
--- a/packages/speech-spine/src/fragment.ts
+++ b/packages/speech-spine/src/fragment.ts
@@ -1,7 +1,6 @@
-import { narrativeTypes } from "@hypit/narrative";
-import { mediaTypes } from "@hypit/media";
import { programSpaceTypes } from "@hypit/program-space";
import { speechTypes } from "@hypit/speech";
+import { semanticMapTypes } from "@hypit/semantic-map";
import { compositionTypes } from "@hypit/composition";
import { sealGraphFragment } from "@hypit/elaborator";
import type { FragmentOperation } from "@hypit/elaborator";
@@ -30,8 +29,7 @@ export function createSpeechSpineFragment(options: SpeechSpineFragmentOptions) {
};
addInput("program", speechSpineTypes.spineProgram);
for (const take of options.takes) {
- addInput(take.mediaName, mediaTypes.synchronized);
- addInput(take.segmentName, narrativeTypes.excerpt);
+ addInput(take.takeName, speechTypes.semanticTake);
if (take.visual !== undefined) {
addInput(take.visual.frameName, spatialTypes.frame);
addInput(take.visual.fitName, spatialTypes.fit);
@@ -53,8 +51,7 @@ export function createSpeechSpineFragment(options: SpeechSpineFragmentOptions) {
inputs: {
set: operation(current),
program: input("program"),
- media: input(take.mediaName),
- segment: input(take.segmentName),
+ take: input(take.takeName),
...(take.visual === undefined ? {} : {
frame: input(take.visual.frameName),
fit: input(take.visual.fitName),
@@ -90,12 +87,6 @@ export function createSpeechSpineFragment(options: SpeechSpineFragmentOptions) {
inputs: { basis: operation("spine:basis") },
result: { kind: "output", name: "programSpace" },
},
- {
- id: "spine:audio-basis",
- producer: speechBasisProducers.projectAudio,
- inputs: { basis: operation("spine:basis") },
- result: { kind: "output", name: "audio" },
- },
{
id: "spine:visual-track",
producer: speechBasisProducers.projectVisual,
@@ -109,6 +100,12 @@ export function createSpeechSpineFragment(options: SpeechSpineFragmentOptions) {
result: { kind: "output", name: "track" },
},
);
+ operations.push({
+ id: "spine:semantic-map",
+ producer: speechSpineProducers.assembleSemanticMap,
+ inputs: { set: operation(current) },
+ result: { kind: "output", name: "map" },
+ });
return sealGraphFragment({
inputs: [...declaredInputs.entries()].map(([name, type]) => ({ name, type })),
operations,
@@ -117,15 +114,13 @@ export function createSpeechSpineFragment(options: SpeechSpineFragmentOptions) {
{
name: "space", type: programSpaceTypes.programSpace, root: operation("spine:space"),
},
- {
- name: "audio", type: speechTypes.audioBasis, root: operation("spine:audio-basis"),
- },
{
name: "visual", type: compositionTypes.visualTrack, root: operation("spine:visual-track"),
},
{
name: "audioTrack", type: compositionTypes.audioTrack, root: operation("spine:audio-track"),
},
+ { name: "semanticMap", type: semanticMapTypes.complete, root: operation("spine:semantic-map") },
],
});
}
diff --git a/packages/speech-spine/src/index.ts b/packages/speech-spine/src/index.ts
index f04a947b..2ab27e7b 100644
--- a/packages/speech-spine/src/index.ts
+++ b/packages/speech-spine/src/index.ts
@@ -3,5 +3,6 @@ export { speechSpineComponent } from "./component.js";
export { createSpeechSpineFragment } from "./fragment.js";
export { speechSpineManifest, speechSpineMarkupSurfaces, speechSpineModuleRef, speechSpineProducers, speechSpineTypes, speechSpineProgramSchema, speechSpineSetSchema, speechSpineVisualSpecSchema } from "./manifest.js";
export { appendSpeechSpineAudioTake, appendSpeechSpineVisualTake, assembleSpeechBasis, assertSpeechSpineProgram, assertSpeechSpineSet, assertSpeechSpineVisualSpec, compileSpeechSpineAudio, createSpeechSpineSet, sealSpeechSpineProgram, sealSpeechSpineVisualSpec } from "./program.js";
+export { assembleSemanticTakeMap } from "./semantic.js";
export { decodeSpeechSpineSurface } from "./surface.js";
export type * from "./types.js";
diff --git a/packages/speech-spine/src/manifest.ts b/packages/speech-spine/src/manifest.ts
index 65a0d370..dd6b667c 100644
--- a/packages/speech-spine/src/manifest.ts
+++ b/packages/speech-spine/src/manifest.ts
@@ -1,10 +1,9 @@
import { readFile } from "node:fs/promises";
import { artifactDependency, artifactTypes } from "@hypit/artifact";
-import { narrativeDependency, narrativeExcerptSchema, narrativeTypes } from "@hypit/narrative";
-import { mediaDependency, mediaTypes, synchronizedMediaSchema } from "@hypit/media";
+import { mediaDependency, mediaTypes } from "@hypit/media";
import { programSpaceDependency, programSpaceTypes } from "@hypit/program-space";
-import { speechDependency, speechTypes } from "@hypit/speech";
+import { semanticTakeSchema, speechDependency, speechTypes } from "@hypit/speech";
import { compositionDependency, compositionTypes } from "@hypit/composition";
import {
mediaPipelineManifest,
@@ -12,6 +11,7 @@ import {
mediaPipelineTypes,
} from "@hypit/media-pipeline";
import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/protocol";
+import { semanticMapDependency, semanticMapTypes } from "@hypit/semantic-map";
import { speechBasisManifest, speechBasisModuleRef } from "@hypit/speech-basis";
import {
contentFitSchema,
@@ -39,6 +39,7 @@ export const speechSpineProducers = {
appendVisualTake: { module: speechSpineModuleRef, name: "append-spine-visual-take" },
compileAudio: { module: speechSpineModuleRef, name: "compile-spine-audio" },
assembleBasis: { module: speechSpineModuleRef, name: "assemble-speech-basis" },
+ assembleSemanticMap: { module: speechSpineModuleRef, name: "assemble-semantic-map" },
} satisfies Record<string, ProducerRef>;
const string = { kind: "string", minLength: 1 } as const;
@@ -58,8 +59,7 @@ export const speechSpineProgramSchema: ValueSchema = object({
export const speechSpineSetSchema: ValueSchema = object({
takes: { schema: { kind: "array", items: object({
- segment: { schema: narrativeExcerptSchema },
- media: { schema: synchronizedMediaSchema },
+ semantic: { schema: semanticTakeSchema },
visual: { schema: object({
frame: { schema: spatialFrameSchema },
fit: { schema: contentFitSchema },
@@ -98,18 +98,19 @@ export const speechSpineMarkupSurfaces = [{
tag: "Spine",
mode: "structured",
outputs: [speechSpineTypes.spineProgram, speechSpineTypes.visualSpec, spatialTypes.fit,
- mediaPipelineTypes.selectionRequest,
- speechTypes.basis, programSpaceTypes.programSpace, speechTypes.audioBasis,
- compositionTypes.visualTrack, compositionTypes.audioTrack],
+ speechTypes.basis, programSpaceTypes.programSpace,
+ compositionTypes.visualTrack, compositionTypes.audioTrack, semanticMapTypes.complete],
vocabulary: {
summary: "Folds ordered speech Takes into one SpeechBasis, and publishes the ProgramSpace every other Track is timed against together with the peer VisualTrack and AudioTrack the speech renders to.",
appearance: "The speaking picture itself, and the layer every other Track is stacked over. Each Take's own footage fills the Frame the Spine names, fitted by its Recipe, and the Takes run one after another in the order they are written, so the picture cuts from one to the next at each Take boundary with nothing between them. A Take may name its own Frame, so the picture can move or resize at a boundary; otherwise the framing holds. Nothing is drawn on top: titles, captions and cutaways are separate Tracks lying above this one.",
preview: previewImage("Spine.png"),
attributes: [
{ name: "id", kind: "identifier", required: true,
- summary: "Names this Spine and prefixes the Program, the media normalization and every binding it publishes." },
- { name: "frame-rate", kind: "literal", required: true,
- summary: "Fixes the frame rate every Take is normalized to and the ProgramSpace is measured in, written as a positive rational such as 30 or 30000/1001." },
+ summary: "Names this Spine and prefixes every projection it publishes." },
+ { name: "clock", kind: "reference", required: false, accepts: [programSpaceTypes.clock],
+ summary: "Selects the authored frame clock shared by the normalized Takes." },
+ { name: "frame-rate", kind: "literal", required: false,
+ summary: "Legacy inline frame rate; write exactly one of clock or frame-rate." },
{ name: "visual-frame", kind: "reference", required: true, accepts: [spatialTypes.frame],
summary: "Chooses the Frame every visual Take occupies unless the Take names its own." },
{ name: "visual-appearance", kind: "reference", required: true, accepts: [svsRecipeType],
@@ -120,16 +121,10 @@ export const speechSpineMarkupSurfaces = [{
],
children: [
{ tag: "Take", cardinality: "many",
- summary: "One spoken Segment and the single source that performs it, in document order.",
+ summary: "One self-contained SemanticTake, in document order.",
attributes: [
- { name: "segment", kind: "reference", required: true, accepts: [narrativeTypes.excerpt],
- summary: "Chooses the spoken Segment this Take performs." },
- { name: "video", kind: "reference", required: false, accepts: [artifactTypes.blob],
- summary: "Performs the Segment from a raw video the Surface inspects, selects and normalizes before assembly." },
- { name: "audio", kind: "reference", required: false, accepts: [artifactTypes.blob],
- summary: "Performs the Segment from a voice recording normalized audio-authoritatively, contributing no visual clip." },
- { name: "media", kind: "reference", required: false, accepts: [mediaTypes.synchronized],
- summary: "Performs the Segment from an already prepared timed source, connected directly." },
+ { name: "source", kind: "reference", required: true, accepts: [speechTypes.semanticTake],
+ summary: "Chooses the normalized and locally aligned SemanticTake this Spine assembles." },
{ name: "frame", kind: "reference", required: false, accepts: [spatialTypes.frame],
summary: "Chooses this Take's own Frame in place of the Spine's `visual-frame`." },
{ name: "appearance", kind: "reference", required: false, accepts: [svsRecipeType],
@@ -144,22 +139,22 @@ export const speechSpineMarkupSurfaces = [{
summary: "The assembled SpeechBasis: the ordered Takes, their placed audio and their visual clips." },
{ name: "space", type: programSpaceTypes.programSpace,
summary: "The speech coordinate space, the frame domain every other Track resolves its windows in." },
- { name: "audio", type: speechTypes.audioBasis,
- summary: "The rendered speech audio, as the timed basis measurement and captioning read." },
{ name: "visual", type: compositionTypes.visualTrack,
summary: "The rendered speech picture, an ordinary peer VisualTrack." },
{ name: "audioTrack", type: compositionTypes.audioTrack,
summary: "The rendered speech sound, an ordinary peer AudioTrack." },
+ { name: "semanticMap", type: semanticMapTypes.complete,
+ summary: "The complete global map assembled by translating each Take's local semantic frames." },
],
- example: `<speech:Spine id="speech" frame-rate="30"
+ example: `<speech:Spine id="speech" clock={clock}
visual-frame={speech-frame} visual-appearance={studio.speech.visual} visual-z="0">
- <speech:Take video={take-opening.video} segment={story.segment.opening}/>
- <speech:Take video={take-closing.video} segment={story.segment.closing}/>
+ <speech:Take source={opening.take}/>
+ <speech:Take source={closing.take}/>
</speech:Spine>`,
notes: [
"A Spine requires at least one Take and accepts no text content.",
"`visual-appearance` and a Take's `appearance` must each be an authored SVS Recipe declaring only the spatial fit properties listed for them; any other property is refused.",
- "A Take states exactly one of `video`, `audio` or `media`; an `audio` Take is refused `frame`, `appearance` and `z`.",
+ "Media normalization, acoustic evidence and Segment alignment happen before a Take enters the Spine.",
"Placement and stacking are the complete visual authority of a Spine; motion, transitions and independent pictures remain ordinary Media Tracks.",
],
},
@@ -172,7 +167,6 @@ export const speechSpineManifest: ModuleManifest = {
version: speechSpineModuleRef.version,
dependencies: [
artifactDependency,
- narrativeDependency,
mediaDependency,
programSpaceDependency,
speechDependency,
@@ -181,6 +175,7 @@ export const speechSpineManifest: ModuleManifest = {
{ module: svsModuleRef },
{ module: mediaPipelineModuleRef },
{ module: speechBasisModuleRef },
+ semanticMapDependency,
],
types: [
{ name: speechSpineTypes.spineProgram.name },
@@ -200,8 +195,7 @@ export const speechSpineManifest: ModuleManifest = {
inputs: [
{ name: "set", type: speechSpineTypes.spineSet },
{ name: "program", type: speechSpineTypes.spineProgram },
- { name: "media", type: mediaTypes.synchronized },
- { name: "segment", type: narrativeTypes.excerpt },
+ { name: "take", type: speechTypes.semanticTake },
],
outputs: [{ name: "set", type: speechSpineTypes.spineSet }],
needs: [],
@@ -211,8 +205,7 @@ export const speechSpineManifest: ModuleManifest = {
inputs: [
{ name: "set", type: speechSpineTypes.spineSet },
{ name: "program", type: speechSpineTypes.spineProgram },
- { name: "media", type: mediaTypes.synchronized },
- { name: "segment", type: narrativeTypes.excerpt },
+ { name: "take", type: speechTypes.semanticTake },
{ name: "frame", type: spatialTypes.frame },
{ name: "fit", type: spatialTypes.fit },
{ name: "visualSpec", type: speechSpineTypes.visualSpec },
@@ -239,5 +232,11 @@ export const speechSpineManifest: ModuleManifest = {
outputs: [{ name: "basis", type: speechTypes.basis }],
needs: [],
},
+ {
+ name: speechSpineProducers.assembleSemanticMap.name,
+ inputs: [{ name: "set", type: speechSpineTypes.spineSet }],
+ outputs: [{ name: "map", type: semanticMapTypes.complete }],
+ needs: [],
+ },
],
};
diff --git a/packages/speech-spine/src/program.ts b/packages/speech-spine/src/program.ts
index df90b58d..2748552c 100644
--- a/packages/speech-spine/src/program.ts
+++ b/packages/speech-spine/src/program.ts
@@ -1,10 +1,9 @@
-import type { NarrativeExcerpt } from "@hypit/narrative";
-import { synchronizedMediaSampleFrames, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
-import type { SynchronizedMedia, TimelineAudio } from "@hypit/media";
+import { synchronizedMediaSampleFrames, verifyTimelineAudio } from "@hypit/media";
+import type { TimelineAudio } from "@hypit/media";
import { assertProgramSpaceIdentity, sealProgramSpace } from "@hypit/program-space";
import type { ProgramSpace } from "@hypit/program-space";
-import { sealSpeechBasis } from "@hypit/speech";
-import type { SpeechBasis } from "@hypit/speech";
+import { assertSemanticTakeIdentity, sealSpeechBasis } from "@hypit/speech";
+import type { SemanticTake, SpeechBasis } from "@hypit/speech";
import {
sealAudioProgramPlan,
verifyAudioProgramPlan,
@@ -44,14 +43,6 @@ export function assertSpeechSpineProgram(value: SpeechSpineProgram): void {
"SpeechSpineProgram frame rate is invalid");
}
-function verifyExcerpt(value: NarrativeExcerpt): void {
- assert(value.kind === "segment",
- "Speech Spine Take must reference a Segment NarrativeExcerpt");
- assert(value.id.length > 0 && Number.isSafeInteger(value.tokenStart)
- && Number.isSafeInteger(value.tokenEndExclusive) && value.tokenEndExclusive > value.tokenStart,
- "Speech Spine Segment excerpt is invalid");
-}
-
function setContent(value: SpeechSpineSet): SpeechSpineSet {
return canonicalize({
@@ -64,19 +55,19 @@ function sealSpeechSpineSet(value: SpeechSpineSet): SpeechSpineSet {
}
function assertTake(take: SpeechSpineTake, program: SpeechSpineProgram): void {
- verifyExcerpt(take.segment);
- verifySynchronizedMedia(take.media);
- assert(take.media.audio !== undefined, `Speech Segment ${take.segment.id} has no speech audio`);
- assert(take.media.timeline.frameRate.numerator === program.frameRate.numerator
- && take.media.timeline.frameRate.denominator === program.frameRate.denominator,
- `Speech Segment ${take.segment.id} uses another frame rate`);
- assert((take.media.visual === undefined) === (take.visual === undefined),
- `Speech Segment ${take.segment.id} visual policy does not match its media`);
+ assertSemanticTakeIdentity(take.semantic);
+ const { media, segment } = take.semantic;
+ assert(media.audio !== undefined, `Speech Segment ${segment.segmentId} has no speech audio`);
+ assert(media.timeline.frameRate.numerator === program.frameRate.numerator
+ && media.timeline.frameRate.denominator === program.frameRate.denominator,
+ `Speech Segment ${segment.segmentId} uses another frame rate`);
+ assert((media.visual === undefined) === (take.visual === undefined),
+ `Speech Segment ${segment.segmentId} visual policy does not match its media`);
if (take.visual !== undefined) {
assertSpatialFrame(take.visual.frame);
assertContentFit(take.visual.fit);
assert(Number.isSafeInteger(take.visual.stackingOrder),
- `Speech Segment ${take.segment.id} visual stacking order is invalid`);
+ `Speech Segment ${segment.segmentId} visual stacking order is invalid`);
}
}
@@ -93,10 +84,10 @@ export function assertSpeechSpineVisualSpec(value: SpeechSpineVisualSpec): void
export function assertSpeechSpineSet(value: SpeechSpineSet): void {
const segments = new Set<string>();
for (const take of value.takes) {
- verifyExcerpt(take.segment);
- verifySynchronizedMedia(take.media);
- assert(!segments.has(take.segment.id), `Speech Spine repeats Segment ${take.segment.id}`);
- segments.add(take.segment.id);
+ assertSemanticTakeIdentity(take.semantic);
+ const segmentId = take.semantic.segment.segmentId;
+ assert(!segments.has(segmentId), `Speech Spine repeats Segment ${segmentId}`);
+ segments.add(segmentId);
}
}
@@ -115,7 +106,8 @@ function appendTake(
assertSpeechSpineSet(set);
assertSpeechSpineProgram(program);
assertTake(take, program);
- assert(!set.takes.some((item) => item.segment.id === take.segment.id), `Speech Spine repeats Segment ${take.segment.id}`);
+ assert(!set.takes.some((item) => item.semantic.segment.segmentId === take.semantic.segment.segmentId),
+ `Speech Spine repeats Segment ${take.semantic.segment.segmentId}`);
return sealSpeechSpineSet({
takes: [...set.takes, take],
@@ -125,19 +117,18 @@ function appendTake(
export function appendSpeechSpineAudioTake(
set: SpeechSpineSet,
program: SpeechSpineProgram,
- media: SynchronizedMedia,
- segment: NarrativeExcerpt,
+ semantic: SemanticTake,
): SpeechSpineSet {
- verifySynchronizedMedia(media);
- assert(media.visual === undefined, `Speech Segment ${segment.id} audio Take unexpectedly contains a visual stream`);
- return appendTake(set, program, { media, segment });
+ assertSemanticTakeIdentity(semantic);
+ assert(semantic.media.visual === undefined,
+ `Speech Segment ${semantic.segment.segmentId} audio Take unexpectedly contains a visual stream`);
+ return appendTake(set, program, { semantic });
}
export function appendSpeechSpineVisualTake(
set: SpeechSpineSet,
program: SpeechSpineProgram,
- media: SynchronizedMedia,
- segment: NarrativeExcerpt,
+ semantic: SemanticTake,
frame: SpatialFrame,
fit: ContentFit,
visualSpec: SpeechSpineVisualSpec,
@@ -145,11 +136,10 @@ export function appendSpeechSpineVisualTake(
assertSpatialFrame(frame);
assertContentFit(fit);
assertSpeechSpineVisualSpec(visualSpec);
- verifySynchronizedMedia(media);
- assert(media.visual !== undefined, `Speech Segment ${segment.id} visual Take has no visual stream`);
+ assertSemanticTakeIdentity(semantic);
+ assert(semantic.media.visual !== undefined, `Speech Segment ${semantic.segment.segmentId} visual Take has no visual stream`);
const take = {
- media,
- segment,
+ semantic,
visual: {
frame: structuredClone(frame),
fit: structuredClone(fit),
@@ -172,7 +162,7 @@ function programSpace(program: SpeechSpineProgram, set: SpeechSpineSet): Program
assertSpeechSpineSet(set);
assert(set.takes.length > 0, "Speech Spine must contain at least one Take");
for (const take of set.takes) assertTake(take, program);
- const frameCount = set.takes.reduce((sum, take) => sum + take.media.timeline.frameCount, 0);
+ const frameCount = set.takes.reduce((sum, take) => sum + take.semantic.media.timeline.frameCount, 0);
const space = sealProgramSpace({
durationSec: frameCount * program.frameRate.denominator / program.frameRate.numerator,
frameRate: { ...program.frameRate },
@@ -185,16 +175,17 @@ export function compileSpeechSpineAudio(program: SpeechSpineProgram, set: Speech
const space = programSpace(program, set);
let frame = 0;
const clips = set.takes.map((take, index) => {
- const audio = take.media.audio!;
+ const { media, segment } = take.semantic;
+ const audio = media.audio!;
const startFrame = frame;
- frame += take.media.timeline.frameCount;
+ frame += media.timeline.frameCount;
const targetStartSample = frameSample(startFrame, program.frameRate);
const targetEndSampleExclusive = frameSample(frame, program.frameRate);
- const sourceSampleFrames = synchronizedMediaSampleFrames(take.media);
+ const sourceSampleFrames = synchronizedMediaSampleFrames(media);
assert(targetEndSampleExclusive - targetStartSample === sourceSampleFrames,
- `Speech Segment ${take.segment.id} audio does not exactly cover its normalized frame span`);
+ `Speech Segment ${segment.segmentId} audio does not exactly cover its normalized frame span`);
return {
- id: `${String(index + 1).padStart(4, "0")}:${take.segment.id}`,
+ id: `${String(index + 1).padStart(4, "0")}:${segment.segmentId}`,
artifact: audio.artifact,
targetStartSample,
targetEndSampleExclusive,
@@ -230,25 +221,26 @@ export function assembleSpeechBasis(
const space = programSpace(program, set);
verifyTimelineAudio(audio);
assert(audio.sampleFrames === frameSample(
- set.takes.reduce((sum, take) => sum + take.media.timeline.frameCount, 0),
+ set.takes.reduce((sum, take) => sum + take.semantic.media.timeline.frameCount, 0),
program.frameRate,
), "TimelineAudio does not cover this Speech Spine");
let frame = 0;
const segments = set.takes.map((take) => {
const startFrame = frame;
- frame += take.media.timeline.frameCount;
- return { segmentId: take.segment.id, startFrame, endFrameExclusive: frame };
+ frame += take.semantic.media.timeline.frameCount;
+ return { segmentId: take.semantic.segment.segmentId, startFrame, endFrameExclusive: frame };
});
frame = 0;
const visualClips = set.takes.flatMap((take) => {
- frame += take.media.timeline.frameCount;
- if (take.media.visual === undefined || take.visual === undefined) return [];
+ const { media, segment } = take.semantic;
+ frame += media.timeline.frameCount;
+ if (media.visual === undefined || take.visual === undefined) return [];
return [{
- segmentId: take.segment.id,
- artifact: structuredClone(take.media.visual.artifact),
+ segmentId: segment.segmentId,
+ artifact: structuredClone(media.visual.artifact),
extent: {
- widthPx: take.media.visual.width,
- heightPx: take.media.visual.height,
+ widthPx: media.visual.width,
+ heightPx: media.visual.height,
},
frame: structuredClone(take.visual.frame),
fit: structuredClone(take.visual.fit),
diff --git a/packages/speech-spine/src/semantic.ts b/packages/speech-spine/src/semantic.ts
new file mode 100644
index 00000000..041fe42d
--- /dev/null
+++ b/packages/speech-spine/src/semantic.ts
@@ -0,0 +1,47 @@
+import type { CompleteSemanticMap, SemanticTimePoint, TimedSpeechToken } from "@hypit/semantic-map";
+import { assertSemanticTakeIdentity } from "@hypit/speech";
+import type { SemanticTake } from "@hypit/speech";
+
+/**
+ * Compose local SemanticTakes in authored Spine order. The input takes already
+ * own their local media frame domains; Spine only translates their coordinates
+ * into one program frame domain and never re-identifies their words.
+ */
+export function assembleSemanticTakeMap(takes: readonly SemanticTake[]): CompleteSemanticMap {
+ const tokens: TimedSpeechToken[] = [];
+ const anchors: SemanticTimePoint[] = [];
+ const segmentIds = new Set<string>();
+ const tokenIds = new Set<string>();
+ const anchorIds = new Set<string>();
+ let frameRate: SemanticTake["media"]["timeline"]["frameRate"] | undefined;
+ let offset = 0;
+ for (const take of takes) {
+ assertSemanticTakeIdentity(take);
+ if (frameRate === undefined) frameRate = take.media.timeline.frameRate;
+ else if (frameRate.numerator !== take.media.timeline.frameRate.numerator
+ || frameRate.denominator !== take.media.timeline.frameRate.denominator) {
+ throw new Error("Semantic Spine Takes must use one frame rate.");
+ }
+ if (segmentIds.has(take.segment.segmentId)) {
+ throw new Error(`Semantic Spine repeats Segment ${take.segment.segmentId}.`);
+ }
+ segmentIds.add(take.segment.segmentId);
+ for (const token of take.tokens) {
+ if (tokenIds.has(token.tokenId)) throw new Error(`Semantic Spine repeats Token ${token.tokenId}.`);
+ tokenIds.add(token.tokenId);
+ tokens.push({
+ tokenId: token.tokenId,
+ segmentId: token.segmentId,
+ startFrame: offset + token.startFrame,
+ endFrameExclusive: offset + token.endFrameExclusive,
+ });
+ }
+ for (const anchor of take.anchors) {
+ if (anchorIds.has(anchor.identity)) throw new Error(`Semantic Spine repeats Anchor ${anchor.identity}.`);
+ anchorIds.add(anchor.identity);
+ anchors.push({ identity: anchor.identity, frame: offset + anchor.frame });
+ }
+ offset += take.media.timeline.frameCount;
+ }
+ return { tokens, anchors };
+}
diff --git a/packages/speech-spine/src/surface.ts b/packages/speech-spine/src/surface.ts
index b1e9ede3..5f9f85f1 100644
--- a/packages/speech-spine/src/surface.ts
+++ b/packages/speech-spine/src/surface.ts
@@ -1,19 +1,16 @@
-import { narrativeTypes } from "@hypit/narrative";
-import { artifactTypes } from "@hypit/artifact";
-import { mediaTypes } from "@hypit/media";
import {
- mediaPipelineTypes,
- sealMediaSelectionRequest,
- synchronizedMediaFragment,
-} from "@hypit/media-pipeline";
+ localName,
+ textAttribute as stringAttribute,
+ type StructuredElement,
+ type StructuredSurfaceHandler,
+ type SurfaceResolvedReference,
+ type MarkupAttributeValue,
+} from "@hypit/markup";
+import { sameType } from "@hypit/protocol";
+import { speechTypes } from "@hypit/speech";
+import { programSpaceTypes, type ProgramClock } from "@hypit/program-space";
import type { SvsRecipe } from "@hypit/svs";
import { svsRecipeType } from "@hypit/svs";
-import type {
- StructuredElement,
- StructuredSurfaceHandler,
- SurfaceResolvedReference,
- MarkupAttributeValue,
-} from "@hypit/markup";
import { createSpeechSpineFragment } from "./fragment.js";
import { speechSpineTypes } from "./manifest.js";
@@ -24,16 +21,6 @@ import {
spatialTypes,
} from "@hypit/spatial";
-function localName(value: string): string {
- return value.includes(":") ? value.slice(value.lastIndexOf(":") + 1) : value;
-}
-
-function exactAttributes(element: StructuredElement, names: readonly string[]): void {
- if (Object.keys(element.attributes).sort().join("\u0000") !== [...names].sort().join("\u0000")) {
- throw new Error(`${element.name} requires exactly ${names.join(", ")}`);
- }
-}
-
function allowedAttributes(element: StructuredElement, required: readonly string[], optional: readonly string[]): void {
const keys = Object.keys(element.attributes);
const missing = required.filter((name) => !keys.includes(name));
@@ -43,12 +30,6 @@ function allowedAttributes(element: StructuredElement, required: readonly string
}
}
-function stringAttribute(element: StructuredElement, name: string): string {
- const value = element.attributes[name];
- if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${element.name}.${name} must be a non-empty string`);
- return value.trim();
-}
-
function frameRate(element: StructuredElement): { readonly numerator: number; readonly denominator: number } {
const value = stringAttribute(element, "frame-rate");
const match = /^(\d+)(?:\/(\d+))?$/u.exec(value);
@@ -76,10 +57,6 @@ function referencePath(element: StructuredElement, name: string): string {
return value.path;
}
-function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
- return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
-}
-
function resolve(
element: StructuredElement,
name: string,
@@ -120,7 +97,10 @@ function takes(element: StructuredElement): StructuredElement[] {
}
export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- exactAttributes(element, ["id", "frame-rate", "visual-frame", "visual-appearance", "visual-z"]);
+ allowedAttributes(element, ["id", "visual-frame", "visual-appearance", "visual-z"], ["clock", "frame-rate"]);
+ const hasClock = element.attributes.clock !== undefined;
+ const hasFrameRate = element.attributes["frame-rate"] !== undefined;
+ if (hasClock === hasFrameRate) throw new Error(`${element.name} requires exactly one of clock or frame-rate.`);
const id = stringAttribute(element, "id");
const baseFrame = resolve(element, "visual-frame", spatialTypes.frame, resolveReference);
const baseAppearanceReference = resolve(element, "visual-appearance", svsRecipeType, resolveReference);
@@ -129,7 +109,13 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
`${element.name}.visual-appearance`,
);
const baseZ = integerAttribute(element, "visual-z");
- const rate = frameRate(element);
+ const rate = hasClock
+ ? (() => {
+ const clock = resolve(element, "clock", programSpaceTypes.clock, resolveReference);
+ if (clock.record?.value.kind !== "inline") throw new Error(`${element.name}.clock must be an authored Clock record.`);
+ return (clock.record.value.value as unknown as ProgramClock).frameRate;
+ })()
+ : frameRate(element);
const frameInputNames = new Map<string, string>();
const frameInputName = (value: SurfaceResolvedReference, fallback: string): string => {
const key = JSON.stringify(value.ref);
@@ -154,17 +140,7 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
return fallback;
};
const declaredTakes = takes(element).map((take, index) => {
- const hasVideo = take.attributes.video !== undefined;
- const hasAudio = take.attributes.audio !== undefined;
- const hasMedia = take.attributes.media !== undefined;
- if (Number(hasVideo) + Number(hasAudio) + Number(hasMedia) !== 1) {
- throw new Error(`${take.name} requires exactly one of video, audio or media`);
- }
- const sourceAttribute = hasVideo ? "video" : hasAudio ? "audio" : "media";
- allowedAttributes(take, [sourceAttribute, "segment"], ["frame", "appearance", "z"]);
- if (hasAudio && (take.attributes.frame !== undefined || take.attributes.appearance !== undefined || take.attributes.z !== undefined)) {
- throw new Error(`${take.name} audio Take cannot declare visual frame, appearance or z`);
- }
+ allowedAttributes(take, ["source"], ["frame", "appearance", "z"]);
const suffix = String(index + 1).padStart(4, "0");
const effectiveFrame = take.attributes.frame === undefined
? baseFrame
@@ -176,7 +152,7 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
? baseAppearance
: recipe(effectiveAppearanceReference, `${take.name}.appearance`);
const effectiveZ = take.attributes.z === undefined ? baseZ : integerAttribute(take, "z");
- const visual = hasAudio ? undefined : {
+ const visual = {
frameName: frameInputName(effectiveFrame, take.attributes.frame === undefined ? "visual-base-frame" : `take-${suffix}-frame`),
fitName: fitInputName(effectiveAppearanceReference, take.attributes.appearance === undefined ? "visual-base-fit" : `take-${suffix}-fit`),
visualSpecName: visualSpecInputName(effectiveZ, take.attributes.z === undefined ? "visual-base-spec" : `take-${suffix}-visual-spec`),
@@ -186,40 +162,22 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
};
return {
suffix,
- mediaName: `take-${suffix}-media`,
- segmentName: `take-${suffix}-segment`,
- sourceKind: hasVideo ? "video" as const : hasAudio ? "audio" as const : "media" as const,
- source: resolve(take, sourceAttribute, hasMedia ? mediaTypes.synchronized : artifactTypes.blob, resolveReference),
- segment: resolve(take, "segment", narrativeTypes.excerpt, resolveReference),
+ takeName: `take-${suffix}`,
+ source: resolve(take, "source", speechTypes.semanticTake, resolveReference),
visual,
range: take.range,
};
});
const programId = `${id}.program`;
- const videoRequestId = `${id}.selection.video`;
- const audioRequestId = `${id}.selection.audio`;
const program = sealSpeechSpineProgram({
id,
frameRate: rate,
});
- const videoRequest = sealMediaSelectionRequest({
- video: { mode: "primary-moving" },
- audio: { mode: "default" },
- spanAuthority: "video",
- frameRate: rate,
- });
- const audioRequest = sealMediaSelectionRequest({
- video: { mode: "none" },
- audio: { mode: "default" },
- spanAuthority: "audio",
- frameRate: rate,
- });
const assembly = createSpeechSpineFragment({
name: `@hypit/speech-spine/surface/${id}@1`,
- takes: declaredTakes.map(({ mediaName, segmentName, visual }) => ({
- mediaName,
- segmentName,
+ takes: declaredTakes.map(({ takeName, visual }) => ({
+ takeName,
...(visual === undefined ? {} : { visual: {
frameName: visual.frameName,
fitName: visual.fitName,
@@ -227,17 +185,6 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
} }),
})),
});
- const normalizationComponents = declaredTakes.flatMap((take, index) => take.sourceKind === "media" ? [] : [{
- id: `${id}.normalize.${String(index + 1).padStart(4, "0")}`,
- fragment: synchronizedMediaFragment.id,
- inputs: {
- source: take.source.ref,
- request: { kind: "record" as const, id: take.sourceKind === "audio" ? audioRequestId : videoRequestId },
- },
- outputs: { media: `${id}.normalized.${String(index + 1).padStart(4, "0")}` },
- range: take.range,
- }]);
- const normalizationByTake = new Map(normalizationComponents.map((component) => [component.id.split(".").at(-1), component]));
const emittedFits = new Set<string>();
const emittedSpecs = new Set<string>();
const visualRecords = declaredTakes.flatMap((take) => {
@@ -269,29 +216,19 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
}
return records;
});
- const hasVideoNormalization = declaredTakes.some((take) => take.sourceKind === "video");
- const hasAudioNormalization = declaredTakes.some((take) => take.sourceKind === "audio");
return {
records: [
{ id: programId, type: speechSpineTypes.spineProgram, value: { kind: "inline", value: program }, range: element.range },
- ...(hasVideoNormalization ? [{ id: videoRequestId, type: mediaPipelineTypes.selectionRequest, value: { kind: "inline" as const, value: videoRequest }, range: element.range }] : []),
- ...(hasAudioNormalization ? [{ id: audioRequestId, type: mediaPipelineTypes.selectionRequest, value: { kind: "inline" as const, value: audioRequest }, range: element.range }] : []),
...visualRecords,
],
components: [
- ...normalizationComponents,
{
id,
fragment: assembly.id,
inputs: {
program: { kind: "record", id: programId },
- ...Object.fromEntries(declaredTakes.flatMap((take, index) => [
- [take.mediaName, take.sourceKind === "media" ? take.source.ref : {
- kind: "component-output" as const,
- component: normalizationByTake.get(String(index + 1).padStart(4, "0"))!.id,
- output: "media",
- }],
- [take.segmentName, take.segment.ref],
+ ...Object.fromEntries(declaredTakes.flatMap((take) => [
+ [take.takeName, take.source.ref],
...(take.visual === undefined ? [] : [
[take.visual.frameName, take.visual.frame.ref],
[take.visual.fitName, { kind: "record" as const, id: `${id}.${take.visual.fitName}` }],
@@ -302,14 +239,17 @@ export const decodeSpeechSpineSurface: StructuredSurfaceHandler = ({ element, re
outputs: {
basis: `${id}.basis`,
space: `${id}.space`,
- audio: `${id}.audio`,
visual: `${id}.visual`,
audioTrack: `${id}.audioTrack`,
+ semanticMap: `${id}.semanticMap`,
},
range: element.range,
},
],
- fragments: [...(normalizationComponents.length === 0 ? [] : [synchronizedMediaFragment]), assembly],
- exports: [`${id}.basis`, `${id}.space`, `${id}.audio`, `${id}.visual`, `${id}.audioTrack`],
+ fragments: [assembly],
+ exports: [
+ `${id}.basis`, `${id}.space`, `${id}.visual`, `${id}.audioTrack`,
+ `${id}.semanticMap`,
+ ],
};
};
diff --git a/packages/speech-spine/src/types.ts b/packages/speech-spine/src/types.ts
index 126f1b9f..ca5c9a9b 100644
--- a/packages/speech-spine/src/types.ts
+++ b/packages/speech-spine/src/types.ts
@@ -1,5 +1,3 @@
-import type { NarrativeExcerpt } from "@hypit/narrative";
-import type { SynchronizedMedia } from "@hypit/media";
import type { ContentFit, SpatialFrame } from "@hypit/spatial";
/** Author-owned policy for one ordered Speech Spine. */
@@ -12,8 +10,7 @@ export type SpeechSpineProgram = {
};
export type SpeechSpineTake = {
- readonly segment: NarrativeExcerpt;
- readonly media: SynchronizedMedia;
+ readonly semantic: import("@hypit/speech").SemanticTake;
readonly visual?: {
readonly frame: SpatialFrame;
readonly fit: ContentFit;
@@ -34,8 +31,7 @@ export type SpeechSpineSet = {
};
export type SpeechSpineInput = {
- readonly mediaName: string;
- readonly segmentName: string;
+ readonly takeName: string;
readonly visual?: {
readonly frameName: string;
readonly fitName: string;
diff --git a/packages/speech-spine/test/program.test.ts b/packages/speech-spine/test/program.test.ts
index 85f2aec0..70046393 100644
--- a/packages/speech-spine/test/program.test.ts
+++ b/packages/speech-spine/test/program.test.ts
@@ -3,7 +3,7 @@ import test from "node:test";
import { fixtureDigest } from "../../../test/fixture-digest.js";
import type { SynchronizedMedia, TimelineAudio } from "@hypit/media";
-import type { NarrativeExcerpt } from "@hypit/narrative";
+import type { SemanticTake } from "@hypit/speech";
import {
appendSpeechSpineAudioTake,
appendSpeechSpineVisualTake,
@@ -22,10 +22,6 @@ const fit = {
constraint: "bounded" as const,
};
-function segment(id: string, index: number): NarrativeExcerpt {
- return { kind: "segment", id, tokenStart: index, tokenEndExclusive: index + 1 };
-}
-
function synchronized(id: string, visual: boolean): SynchronizedMedia {
return {
timeline: {
@@ -43,6 +39,35 @@ function synchronized(id: string, visual: boolean): SynchronizedMedia {
};
}
+function semantic(id: string, visual: boolean): SemanticTake {
+ const media = synchronized(id, visual);
+ return {
+ media,
+ segment: {
+ segmentId: id,
+ startAnchorId: `segment:${id}:start`,
+ endAnchorId: `segment:${id}:end`,
+ startFrame: 0,
+ endFrameExclusive: media.timeline.frameCount,
+ },
+ tokens: [{
+ tokenId: `segment:${id}:token:1`,
+ segmentId: id,
+ text: id,
+ startAnchorId: `segment:${id}:token:1:start`,
+ endAnchorId: `segment:${id}:token:1:end`,
+ startFrame: 0,
+ endFrameExclusive: media.timeline.frameCount,
+ }],
+ anchors: [
+ { identity: `segment:${id}:start`, frame: 0 },
+ { identity: `segment:${id}:end`, frame: media.timeline.frameCount },
+ { identity: `segment:${id}:token:1:start`, frame: 0 },
+ { identity: `segment:${id}:token:1:end`, frame: media.timeline.frameCount },
+ ],
+ };
+}
+
test("audio Takes lengthen the speech program without inventing a visual clip", () => {
const program = sealSpeechSpineProgram({
@@ -50,12 +75,11 @@ test("audio Takes lengthen the speech program without inventing a visual clip",
frameRate: { numerator: 30, denominator: 1 },
});
let set = createSpeechSpineSet();
- set = appendSpeechSpineAudioTake(set, program, synchronized("voice", false), segment("voiceover", 0));
+ set = appendSpeechSpineAudioTake(set, program, semantic("voiceover", false));
set = appendSpeechSpineVisualTake(
set,
program,
- synchronized("presenter", true),
- segment("answer", 1),
+ semantic("answer", true),
frame,
fit,
sealSpeechSpineVisualSpec({ stackingOrder: 30 }),
@@ -70,7 +94,7 @@ test("audio Takes lengthen the speech program without inventing a visual clip",
assert.equal(basis.visualTrack.clips.length, 1);
assert.deepEqual(basis.visualTrack.clips[0], {
segmentId: "answer",
- artifact: synchronized("presenter", true).visual!.artifact,
+ artifact: synchronized("answer", true).visual!.artifact,
extent: { widthPx: 720, heightPx: 1280 },
frame,
fit,
diff --git a/packages/speech-spine/test/surface.test.ts b/packages/speech-spine/test/surface.test.ts
index d393e353..fa35f5c1 100644
--- a/packages/speech-spine/test/surface.test.ts
+++ b/packages/speech-spine/test/surface.test.ts
@@ -1,160 +1,98 @@
-import { artifactDependency } from "@hypit/artifact";
-import { videoContractManifests } from "../../../test/support/video-domain.js";
-import { registerTypeValidatorFacets } from "@hypit/component-kit";
-import { mediaDependency } from "@hypit/media";
-import { compositionTypes } from "@hypit/composition";
-import { artifactTypes } from "@hypit/artifact";
import assert from "node:assert/strict";
import test from "node:test";
import { fixtureDigest } from "../../../test/fixture-digest.js";
-import { createResolvedClosure, sealBuildRequest, start } from "@hypit/core";
-import {
- AuthorFrontendRegistry,
- compileSourceClosure,
- resolveCompiledSourceExport,
-} from "@hypit/elaborator";
-import type { AuthorSourceUnit } from "@hypit/elaborator";
-import {
- mediaPipelineComponent,
- mediaPipelineManifest,
- mediaPipelineProducers,
-} from "@hypit/media-pipeline";
-import type { ModuleManifest } from "@hypit/protocol";
-import {
- decodeScriptSurface,
- scriptManifest,
- scriptMarkupSurfaces,
- scriptModuleRef,
-} from "@hypit/script";
-import { speechBasisManifest, speechBasisProducers } from "@hypit/speech-basis";
-import { textComponent, textManifest } from "@hypit/text";
-import { mediaTrackManifest } from "@hypit/media-track";
-import { svsManifest, svsModuleRef, svsRecipeType } from "@hypit/svs";
-import {
- decodeSpeechSpineSurface,
- speechSpineManifest,
- speechSpineMarkupSurfaces,
- speechSpineModuleRef,
- speechSpineProducers,
-} from "@hypit/speech-spine";
-import {
- decodeCanvasSurface,
- decodeFrameSurface,
- spatialComponent,
- spatialMarkupSurfaces,
- spatialModuleRef,
-} from "@hypit/spatial";
-import {
- MarkupSurfaceRegistry,
- createMarkupAuthorFrontend,
-} from "@hypit/markup";
-import { createRecordAdmitter, TypeValidatorRegistry } from "@hypit/validation";
+import type { StructuredElement, SurfaceResolvedReference } from "@hypit/markup";
+import type { SemanticTake } from "@hypit/speech";
+import { speechTypes } from "@hypit/speech";
+import { spatialTypes } from "@hypit/spatial";
+import { svsRecipeType } from "@hypit/svs";
+import { decodeSpeechSpineSurface } from "@hypit/speech-spine";
-const fixtureModule = { name: "example.speech-media", version: "1" } as const;
-const fixtureSurfaceDigest = fixtureDigest("example.speech-media/surface@1");
-const fixtureSurface = {
- name: "media", tag: "Media", mode: "structured", outputs: [artifactTypes.blob, svsRecipeType],
-} as const;
-const fixtureManifest: ModuleManifest = {
- format: "hypit.module@1",
- name: fixtureModule.name,
- version: fixtureModule.version,
- dependencies: [artifactDependency, mediaDependency, { module: svsModuleRef }],
- types: [], capabilities: [], producers: [],
+const take: SemanticTake = {
+ media: {
+ timeline: { frameRate: { numerator: 30, denominator: 1 }, frameCount: 30 },
+ visual: {
+ artifact: { kind: "blob", digest: fixtureDigest("spine:take:video"), size: 1, mediaType: "video/mp4" },
+ width: 720,
+ height: 1280,
+ },
+ audio: {
+ artifact: { kind: "blob", digest: fixtureDigest("spine:take:audio"), size: 1, mediaType: "audio/wav" },
+ },
+ },
+ segment: {
+ segmentId: "opening",
+ startAnchorId: "opening:start",
+ endAnchorId: "opening:end",
+ startFrame: 0,
+ endFrameExclusive: 30,
+ },
+ tokens: [],
+ anchors: [
+ { identity: "opening:start", frame: 0 },
+ { identity: "opening:end", frame: 30 },
+ ],
};
-function source(text: string): AuthorSourceUnit {
- return {
- id: "/project/main.svml",
- name: "main.svml",
- text: `<?svml using="@hypit/markup@1"?>\n${text}`,
+function resolved(path: string): SurfaceResolvedReference | undefined {
+ if (path === "opening.take") return {
+ path,
+ ref: { kind: "record", id: path },
+ type: speechTypes.semanticTake,
+ record: { id: path, type: speechTypes.semanticTake, value: { kind: "inline", value: take } },
+ };
+ if (path === "full") return {
+ path,
+ ref: { kind: "record", id: path },
+ type: spatialTypes.frame,
+ record: { id: path, type: spatialTypes.frame, value: { kind: "inline", value: {
+ xPx: 0, yPx: 0, widthPx: 720, heightPx: 1280,
+ } } },
};
+ if (path === "speech-style") return {
+ path,
+ ref: { kind: "record", id: path },
+ type: svsRecipeType,
+ record: { id: path, type: svsRecipeType, value: { kind: "inline", value: {
+ path: "speech.base", properties: { fit: "cover" },
+ } } },
+ };
+ return undefined;
}
-test("Speech Spine lowers ordered Takes into media normalization, one audio plan and peer projections", async () => {
- const closure = createResolvedClosure([
- ...videoContractManifests,
- mediaPipelineManifest,
- mediaTrackManifest,
- svsManifest,
- speechBasisManifest,
- speechSpineManifest,
- textManifest,
- scriptManifest,
- fixtureManifest,
- ]);
- const surfaces = new MarkupSurfaceRegistry();
- surfaces.registerRaw({ module: scriptModuleRef, declaration: scriptMarkupSurfaces[0]!, handler: decodeScriptSurface });
- surfaces.registerStructured({ module: speechSpineModuleRef, declaration: speechSpineMarkupSurfaces.find((item) => item.name === "spine")!, handler: decodeSpeechSpineSurface });
- surfaces.registerStructured({ module: spatialModuleRef, declaration: spatialMarkupSurfaces.find((item) => item.name === "canvas")!, handler: decodeCanvasSurface });
- surfaces.registerStructured({ module: spatialModuleRef, declaration: spatialMarkupSurfaces.find((item) => item.name === "frame")!, handler: decodeFrameSurface });
- surfaces.registerStructured({ module: fixtureModule, declaration: fixtureSurface, handler: ({ element }) => ({
- records: [
- ...["take-one", "take-two", "voice-one"].map((id) => ({
- id, type: artifactTypes.blob,
- value: { kind: "blob" as const, digest: fixtureDigest(id), size: 128,
- mediaType: id === "voice-one" ? "audio/mpeg" : "video/mp4" },
- range: element.range,
- })),
- {
- id: "speech-style", type: svsRecipeType,
- value: { kind: "inline" as const, value: {
- path: "speech.base", properties: { fit: "cover" },
- } },
- range: element.range,
- },
- ],
- components: [], fragments: [],
- }) });
- const frontends = new AuthorFrontendRegistry();
- frontends.register(createMarkupAuthorFrontend({
- registry: surfaces,
- resolveModule(request) {
- if (request.from.startsWith("@hypit/script")) return scriptModuleRef;
- if (request.from.startsWith("@hypit/speech")) return speechSpineModuleRef;
- if (request.from.startsWith("@hypit/spatial")) return spatialModuleRef;
- return fixtureModule;
+test("Speech Spine accepts only already-semantic Segment Takes", async () => {
+ const element: StructuredElement = {
+ kind: "element",
+ name: "speech:Spine",
+ attributes: {
+ id: "speech",
+ "frame-rate": "30",
+ "visual-frame": { kind: "reference", path: "full" },
+ "visual-appearance": { kind: "reference", path: "speech-style" },
+ "visual-z": "0",
},
- }));
- const validators = new TypeValidatorRegistry();
- registerTypeValidatorFacets(validators, mediaPipelineComponent.validators ?? []);
- registerTypeValidatorFacets(validators, spatialComponent.validators ?? []);
- registerTypeValidatorFacets(validators, textComponent.validators ?? []);
- const compiled = await compileSourceClosure({
- entry: source(`<svml>
- <import from="@hypit/script@1"/>
- <import as="fixture" from="example.speech-media@1"/>
- <import as="speech" from="@hypit/speech-spine@1"/>
- <import as="space" from="@hypit/spatial@1"/>
- <script id="story">
- <opening><ALICE> Hello from Alice.</opening>
- <answer><BOB> Hello from Bob.</answer>
- </script>
- <fixture:Media/>
- <space:Canvas id="vertical" width="720" height="1280"/>
- <space:Frame id="speech-frame" within={vertical} left="0%" top="0%" right="100%" bottom="100%"/>
- <speech:Spine id="speech" frame-rate="30"
- visual-frame={speech-frame} visual-appearance={speech-style} visual-z="0">
- <speech:Take audio={voice-one} segment={story.segment.opening}/>
- <speech:Take video={take-two} segment={story.segment.answer}/>
- </speech:Spine>
- </svml>`),
- closure,
- frontends,
- admitRecord: createRecordAdmitter(validators),
- resolveSource() { throw new Error("fixture has no source imports"); },
+ children: [{
+ kind: "element",
+ name: "speech:Take",
+ attributes: { source: { kind: "reference", path: "opening.take" } },
+ children: [],
+ range: { start: 20, end: 60 },
+ }],
+ range: { start: 0, end: 70 },
+ };
+ const output = await decodeSpeechSpineSurface({
+ sourceName: "main.svml",
+ element,
+ resolveReference: resolved,
+ resolveAsset() { throw new Error("Speech Spine does not resolve assets"); },
+ });
+ assert.deepEqual(output.components[0]?.outputs, {
+ basis: "speech.basis",
+ space: "speech.space",
+ visual: "speech.visual",
+ audioTrack: "speech.audioTrack",
+ semanticMap: "speech.semanticMap",
});
- const target = resolveCompiledSourceExport(compiled, "speech.visual", compositionTypes.visualTrack);
- const build = start(compiled.program, compiled.graph, sealBuildRequest({
- targets: [{ output: target.ref.kind === "logical-output" ? target.ref.id : "" }],
- }));
- const names = build.plan.steps.map((step) => step.producer.name);
- assert.equal(names.filter((name) => name === mediaPipelineProducers.inspect.name).length, 2);
- assert.equal(names.filter((name) => name === mediaPipelineProducers.normalize.name).length, 2);
- assert.equal(names.filter((name) => name === speechSpineProducers.appendAudioTake.name).length, 1);
- assert.equal(names.filter((name) => name === speechSpineProducers.appendVisualTake.name).length, 1);
- assert.equal(names.filter((name) => name === speechSpineProducers.compileAudio.name).length, 1);
- assert.equal(names.filter((name) => name === mediaPipelineProducers.renderAudio.name).length, 1);
- assert.equal(names.filter((name) => name === speechBasisProducers.projectVisual.name).length, 1);
+ assert.equal(output.exports?.includes("speech.audio") ?? false, false);
});
diff --git a/packages/speech/README.md b/packages/speech/README.md
index 7d4058b4..298cc124 100644
--- a/packages/speech/README.md
+++ b/packages/speech/README.md
@@ -1,6 +1,7 @@
# `@hypit/speech`
-Public contracts for speech duration, assembled bases and alignment audio projections. SpeechBasis
-Segments use exact ProgramSpace frame boundaries. The separate SpeechEvidenceAudio contract carries
-canonical 16 kHz WAV bytes and their exact sample count; neither contract duplicates floating-point
-Segment seconds.
+Public contracts for speech duration, normalized semantic Takes, assembled speech bases and provider-
+neutral evidence audio. A `SemanticTake` contains one normalized media product, one authored Segment,
+its words and local frame anchors. `SpeechBasis` is only the assembled post-Spine product used to
+project final peer VisualTrack and AudioTrack values. `SpeechEvidenceAudio` carries canonical 16 kHz
+WAV bytes and their exact sample count; it never carries Script or Segment identity.
diff --git a/packages/speech/package.json b/packages/speech/package.json
index bc2d3796..a730f087 100644
--- a/packages/speech/package.json
+++ b/packages/speech/package.json
@@ -12,9 +12,9 @@
},
"dependencies": {
"@hypit/media": "workspace:*",
- "@hypit/narrative": "workspace:*",
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
+ "@hypit/semantic-map": "workspace:*",
"@hypit/spatial": "workspace:*"
}
}
diff --git a/packages/speech/src/identity.ts b/packages/speech/src/identity.ts
index 5c5c0954..740d8243 100644
--- a/packages/speech/src/identity.ts
+++ b/packages/speech/src/identity.ts
@@ -1,7 +1,8 @@
import { isDigest } from "@hypit/protocol";
import { assertProgramSpaceIdentity, programSpaceFrameCount } from "@hypit/program-space";
+import { verifySynchronizedMedia } from "@hypit/media";
import { assertContentFit, assertIntrinsicExtent, assertSpatialFrame } from "@hypit/spatial";
-import type { SpeechAudioBasis, SpeechBasis, SpeechDuration, SpeechEvidenceAudio } from "./types.js";
+import type { SemanticTake, SpeechBasis, SpeechDuration, SpeechEvidenceAudio } from "./types.js";
function assertAudioBlob(value: SpeechBasis["audio"], label: string): void {
if (value.kind !== "blob" || !isDigest(value.digest) || !Number.isSafeInteger(value.size)
|| value.size < 0 || value.mediaType !== "audio/wav") throw new Error(`${label} must be a canonical WAV BlobRef.`);
@@ -18,6 +19,43 @@ export function speechEvidenceSampleBoundary(masterSampleBoundary: number): numb
}
export function sealSpeechBasis(value: SpeechBasis): SpeechBasis { return structuredClone(value); }
export function sealSpeechEvidenceAudio(value: SpeechEvidenceAudio): SpeechEvidenceAudio { return structuredClone(value); }
+
+export function sealSemanticTake(value: SemanticTake): SemanticTake { return structuredClone(value); }
+
+export function assertSemanticTakeIdentity(take: SemanticTake): void {
+ verifySynchronizedMedia(take.media);
+ const frameCount = take.media.timeline.frameCount;
+ const { segment } = take;
+ if (!segment.segmentId || !segment.startAnchorId || !segment.endAnchorId
+ || segment.startFrame !== 0 || segment.endFrameExclusive !== frameCount) {
+ throw new Error("SemanticTake Segment must cover its local media frame domain.");
+ }
+ const tokenIds = new Set<string>();
+ const referencedAnchorIds = new Set<string>([segment.startAnchorId, segment.endAnchorId]);
+ for (const token of take.tokens) {
+ if (!token.tokenId || !token.segmentId || token.segmentId !== segment.segmentId || !token.text
+ || !token.startAnchorId || !token.endAnchorId || tokenIds.has(token.tokenId)
+ || !Number.isSafeInteger(token.startFrame) || !Number.isSafeInteger(token.endFrameExclusive)
+ || token.startFrame < 0 || token.endFrameExclusive < 0
+ || token.startFrame > frameCount || token.endFrameExclusive > frameCount) {
+ throw new Error(`SemanticTake token ${token.tokenId || "<unnamed>"} is invalid.`);
+ }
+ referencedAnchorIds.add(token.startAnchorId);
+ referencedAnchorIds.add(token.endAnchorId);
+ tokenIds.add(token.tokenId);
+ }
+ const anchorIds = new Set<string>();
+ for (const anchor of take.anchors) {
+ if (!anchor.identity || anchorIds.has(anchor.identity) || !Number.isSafeInteger(anchor.frame)
+ || anchor.frame < 0 || anchor.frame > frameCount) {
+ throw new Error(`SemanticTake anchor ${anchor.identity || "<unnamed>"} is invalid.`);
+ }
+ anchorIds.add(anchor.identity);
+ }
+ for (const anchorId of referencedAnchorIds) {
+ if (!anchorIds.has(anchorId)) throw new Error(`SemanticTake is missing referenced Anchor ${anchorId}.`);
+ }
+}
export function assertSpeechEvidenceAudioIdentity(value: SpeechEvidenceAudio): void {
if (value.artifact.kind !== "blob" || !isDigest(value.artifact.digest)
|| !Number.isSafeInteger(value.artifact.size) || value.artifact.size < 0 || value.artifact.mediaType !== "audio/wav"
@@ -60,18 +98,3 @@ export function assertSpeechBasisIdentity(basis: SpeechBasis): void {
assertContentFit(clip.fit);
}
}
-export function assertSpeechAudioBasisIdentity(basis: SpeechAudioBasis): void {
- assertProgramSpaceIdentity(basis.programSpace);
- assertAudioBlob(basis.audio, "SpeechAudioBasis audio");
- if (basis.segments.length === 0) throw new Error("SpeechAudioBasis must contain at least one Segment.");
- let previousEnd = 0;
- const frameCount = programSpaceFrameCount(basis.programSpace);
- for (const segment of basis.segments) {
- if (!segment.segmentId || !Number.isSafeInteger(segment.startFrame)
- || !Number.isSafeInteger(segment.endFrameExclusive) || segment.startFrame < previousEnd
- || segment.endFrameExclusive < segment.startFrame || segment.endFrameExclusive > frameCount) {
- throw new Error("SpeechAudioBasis Segment is invalid or out of order.");
- }
- previousEnd = segment.endFrameExclusive;
- }
-}
diff --git a/packages/speech/src/index.ts b/packages/speech/src/index.ts
index 57f21b2a..b79dda67 100644
--- a/packages/speech/src/index.ts
+++ b/packages/speech/src/index.ts
@@ -1,4 +1,4 @@
-export { assertSpeechAudioBasisIdentity, assertSpeechBasisIdentity, assertSpeechDurationIdentity, assertSpeechEvidenceAudioIdentity, sealSpeechBasis, sealSpeechDuration, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "./identity.js";
+export { assertSemanticTakeIdentity, assertSpeechBasisIdentity, assertSpeechDurationIdentity, assertSpeechEvidenceAudioIdentity, sealSemanticTake, sealSpeechBasis, sealSpeechDuration, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "./identity.js";
export { speechDependency, speechManifest, speechModuleRef, speechTypes } from "./manifest.js";
-export { speechAudioBasisSchema, speechBasisSchema, speechDurationSchema, speechEvidenceAudioSchema } from "./schema.js";
+export { semanticTakeSchema, speechBasisSchema, speechDurationSchema, speechEvidenceAudioSchema } from "./schema.js";
export type * from "./types.js";
diff --git a/packages/speech/src/manifest.ts b/packages/speech/src/manifest.ts
index 4eb2323c..dba40827 100644
--- a/packages/speech/src/manifest.ts
+++ b/packages/speech/src/manifest.ts
@@ -1,17 +1,18 @@
import { mediaDependency } from "@hypit/media";
-import { narrativeDependency } from "@hypit/narrative";
import { programSpaceDependency } from "@hypit/program-space";
import type { ModuleManifest, TypeRef } from "@hypit/protocol";
import { spatialDependency } from "@hypit/spatial";
-import { speechAudioBasisSchema, speechBasisSchema, speechDurationSchema, speechEvidenceAudioSchema } from "./schema.js";
+import { semanticTakeSchema, speechBasisSchema, speechDurationSchema, speechEvidenceAudioSchema } from "./schema.js";
export const speechModuleRef = { name: "@hypit/speech", version: "1" } as const;
export const speechTypes = {
duration: { module: speechModuleRef, name: "SpeechDuration" }, basis: { module: speechModuleRef, name: "SpeechBasis" },
- audioBasis: { module: speechModuleRef, name: "SpeechAudioBasis" }, evidenceAudio: { module: speechModuleRef, name: "SpeechEvidenceAudio" },
+ evidenceAudio: { module: speechModuleRef, name: "SpeechEvidenceAudio" },
+ semanticTake: { module: speechModuleRef, name: "SemanticTake" },
} satisfies Record<string, TypeRef>;
export const speechManifest: ModuleManifest = { format: "hypit.module@1", name: speechModuleRef.name, version: speechModuleRef.version,
- dependencies: [narrativeDependency, mediaDependency, programSpaceDependency, spatialDependency],
+ dependencies: [mediaDependency, programSpaceDependency, spatialDependency],
types: [{ name: speechTypes.duration.name }, { name: speechTypes.basis.name },
- { name: speechTypes.audioBasis.name }, { name: speechTypes.evidenceAudio.name }],
+ { name: speechTypes.evidenceAudio.name },
+ { name: speechTypes.semanticTake.name }],
capabilities: [], producers: [] };
export const speechDependency = { module: speechModuleRef } as const;
diff --git a/packages/speech/src/schema.ts b/packages/speech/src/schema.ts
index 155a344d..998a1e57 100644
--- a/packages/speech/src/schema.ts
+++ b/packages/speech/src/schema.ts
@@ -1,4 +1,5 @@
import { programSpaceSchema } from "@hypit/program-space";
+import { synchronizedMediaSchema } from "@hypit/media";
import type { ValueSchema } from "@hypit/protocol";
import { contentFitSchema, intrinsicExtentSchema, spatialFrameSchema } from "@hypit/spatial";
const string = { kind: "string", minLength: 1 } as const;
@@ -17,6 +18,23 @@ const segment = object({
startFrame: { schema: integer },
endFrameExclusive: { schema: integer },
});
+const semanticTakeToken = object({
+ tokenId: { schema: string },
+ segmentId: { schema: string },
+ text: { schema: string },
+ startAnchorId: { schema: string },
+ endAnchorId: { schema: string },
+ startFrame: { schema: integer },
+ endFrameExclusive: { schema: integer },
+});
+const semanticTakeAnchor = object({ identity: { schema: string }, frame: { schema: integer } });
+const semanticTakeSegment = object({
+ segmentId: { schema: string },
+ startAnchorId: { schema: string },
+ endAnchorId: { schema: string },
+ startFrame: { schema: integer },
+ endFrameExclusive: { schema: integer },
+});
export const speechDurationSchema: ValueSchema = number;
export const speechBasisSchema: ValueSchema = object({
programSpace: { schema: programSpaceSchema }, audio: { schema: audioBlobRef },
@@ -28,9 +46,13 @@ export const speechBasisSchema: ValueSchema = object({
}) } } }) },
segments: { schema: { kind: "array", minItems: 1, items: segment } },
});
-export const speechAudioBasisSchema: ValueSchema = object({
- programSpace: { schema: programSpaceSchema }, audio: { schema: audioBlobRef }, segments: { schema: { kind: "array", minItems: 1, items: segment } } });
export const speechEvidenceAudioSchema: ValueSchema = object({
artifact: { schema: audioBlobRef },
sampleFrames: { schema: { kind: "number", integer: true, minimum: 1 } },
});
+export const semanticTakeSchema: ValueSchema = object({
+ media: { schema: synchronizedMediaSchema },
+ segment: { schema: semanticTakeSegment },
+ tokens: { schema: { kind: "array", items: semanticTakeToken } },
+ anchors: { schema: { kind: "array", items: semanticTakeAnchor } },
+});
diff --git a/packages/speech/src/types.ts b/packages/speech/src/types.ts
index 8c60516e..ea1c2acf 100644
--- a/packages/speech/src/types.ts
+++ b/packages/speech/src/types.ts
@@ -1,6 +1,8 @@
import type { ProgramSpace } from "@hypit/program-space";
import type { BlobRef } from "@hypit/protocol";
+import type { SynchronizedMedia } from "@hypit/media";
import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@hypit/spatial";
+import type { SemanticTimePoint } from "@hypit/semantic-map";
/** Positive seconds. The graph port's TypeRef already carries the SpeechDuration identity. */
export type SpeechDuration = number;
export type SpeechBasisSegment = {
@@ -20,12 +22,37 @@ export type SpeechBasis = {
}[] };
readonly segments: readonly SpeechBasisSegment[];
};
-export type SpeechAudioBasis = {
- readonly programSpace: ProgramSpace; readonly audio: BlobRef;
- readonly segments: readonly SpeechBasisSegment[];
-};
export type SpeechEvidenceAudio = {
readonly artifact: BlobRef;
/** Exact 16 kHz mono PCM sample count. Format constants belong to this Type, not every value. */
readonly sampleFrames: number;
};
+
+/** One authored Script token located in one normalized Take's local frame domain. */
+export type SemanticTakeToken = {
+ readonly tokenId: string;
+ readonly segmentId: string;
+ readonly text: string;
+ readonly startAnchorId: string;
+ readonly endAnchorId: string;
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+};
+
+/**
+ * A self-contained semantic media product. Media bytes remain ordinary BlobRefs
+ * inside SynchronizedMedia; the authored word surface and its local frame
+ * coordinates travel with this Take instead of requiring a second Script edge.
+ */
+export type SemanticTake = {
+ readonly media: SynchronizedMedia;
+ readonly segment: {
+ readonly segmentId: string;
+ readonly startAnchorId: string;
+ readonly endAnchorId: string;
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+ };
+ readonly tokens: readonly SemanticTakeToken[];
+ readonly anchors: readonly SemanticTimePoint[];
+};
diff --git a/packages/whisperx/README.md b/packages/whisperx/README.md
index 6e3493e1..4c220478 100644
--- a/packages/whisperx/README.md
+++ b/packages/whisperx/README.md
@@ -8,11 +8,13 @@ The package contains no credentials, Python environment or queue. Providers tran
request directly into provider-neutral `AlignedTranscriptEvidence`. There is no vendor-shaped
Evidence wrapper or pass-through normalization node in the graph.
-The request Producer consumes only canonical `SpeechEvidenceAudio` bytes. `@hypit/media-pipeline`
-first requests the explicit 48 kHz stereo speech-master → 16 kHz mono evidence projection. Authored
-Segment identity never enters the WhisperX Need or Provider. The later Speech Alignment operation
-receives `SpeechAudioBasis` through its own graph edge and assigns acoustic passages to its exact
-Segment frame windows there.
+`<whisperx:SemanticTake>` consumes one normalized `SynchronizedMedia` and exactly one Script
+Segment. `@hypit/media-pipeline` projects that Take's 48 kHz audio to canonical 16 kHz mono
+`SpeechEvidenceAudio`; WhisperX sees only those bytes. A deterministic local alignment then combines
+the returned evidence with the one Segment and emits one self-contained `SemanticTake`.
+
+There is no whole-program WhisperX pass. Speech Spine only receives already-semantic Takes and later
+translates their local frames when assembling the final ProgramSpace and complete semantic map.
`@hypit/provider-whisperx-local` is the first concrete adapter; it validates and stages those bytes
unchanged for a warm loopback service.
diff --git a/packages/whisperx/package.json b/packages/whisperx/package.json
index 3d40e431..fd25a7d0 100644
--- a/packages/whisperx/package.json
+++ b/packages/whisperx/package.json
@@ -14,9 +14,9 @@
"@hypit/component-kit": "workspace:*",
"@hypit/elaborator": "workspace:*",
"@hypit/media-pipeline": "workspace:*",
+ "@hypit/media": "workspace:*",
"@hypit/narrative": "workspace:*",
"@hypit/protocol": "workspace:*",
- "@hypit/semantic-map": "workspace:*",
"@hypit/speech": "workspace:*",
"@hypit/speech-alignment": "workspace:*",
"@hypit/speech-evidence": "workspace:*",
diff --git a/packages/whisperx/src/activation.ts b/packages/whisperx/src/activation.ts
index dcf13646..4b2d223a 100644
--- a/packages/whisperx/src/activation.ts
+++ b/packages/whisperx/src/activation.ts
@@ -1,6 +1,6 @@
import { createMarkupSurfaceHostFacet } from "@hypit/markup";
import {
- decodeWhisperXAlignmentSurface, whisperXComponent,
+ decodeWhisperXSemanticTakeSurface, whisperXComponent,
whisperXManifest, whisperXModuleRef,
whisperXMarkupSurfaces,
} from "./index.js";
@@ -11,8 +11,8 @@ export const hypitPackage = {
components: [whisperXComponent],
hostFacets: [createMarkupSurfaceHostFacet({
module: whisperXModuleRef,
- declaration: whisperXMarkupSurfaces.find((item) => item.name === "alignment")!,
- handler: decodeWhisperXAlignmentSurface,
+ declaration: whisperXMarkupSurfaces.find((item) => item.name === "semantic-take")!,
+ handler: decodeWhisperXSemanticTakeSurface,
})],
};
export default hypitPackage;
diff --git a/packages/whisperx/src/fragment.ts b/packages/whisperx/src/fragment.ts
index e148929f..f6bc0969 100644
--- a/packages/whisperx/src/fragment.ts
+++ b/packages/whisperx/src/fragment.ts
@@ -1,7 +1,7 @@
import { narrativeTypes } from "@hypit/narrative";
+import { mediaTypes } from "@hypit/media";
import { speechTypes } from "@hypit/speech";
import { speechEvidenceTypes } from "@hypit/speech-evidence";
-import { semanticMapTypes } from "@hypit/semantic-map";
import { sealGraphFragment } from "@hypit/elaborator";
import { mediaPipelineProducers } from "@hypit/media-pipeline";
import { speechAlignmentProducers } from "@hypit/speech-alignment";
@@ -11,17 +11,18 @@ import { whisperXProducers } from "./manifest.js";
const input = (name: string) => ({ kind: "fragment-input" as const, name });
const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
-/** One measured acoustic pass followed by provider-neutral deterministic Script alignment. */
-export const whisperXSpeechAlignmentFragment = sealGraphFragment({
+/** Normalize one Take's audio evidence, measure it, then project one Script Segment locally. */
+export const whisperXSemanticTakeFragment = sealGraphFragment({
inputs: [
{ name: "narrative", type: narrativeTypes.narrative },
- { name: "audio", type: speechTypes.audioBasis },
+ { name: "segment", type: narrativeTypes.excerpt },
+ { name: "media", type: mediaTypes.synchronized },
],
operations: [
{
id: "prepare-evidence-audio",
producer: mediaPipelineProducers.projectSpeechEvidenceAudio,
- inputs: { audio: input("audio") },
+ inputs: { media: input("media") },
result: { kind: "need", name: "evidenceAudio" },
},
{
@@ -31,10 +32,15 @@ export const whisperXSpeechAlignmentFragment = sealGraphFragment({
result: { kind: "need", name: "alignment" },
},
{
- id: "locate-speech",
- producer: speechAlignmentProducers.locate,
- inputs: { narrative: input("narrative"), audio: input("audio"), evidence: operation("request-whisperx") },
- result: { kind: "output", name: "map" },
+ id: "align-semantic-take",
+ producer: speechAlignmentProducers.alignTake,
+ inputs: {
+ narrative: input("narrative"),
+ segment: input("segment"),
+ media: input("media"),
+ evidence: operation("request-whisperx"),
+ },
+ result: { kind: "output", name: "take" },
},
],
exports: [
@@ -44,9 +50,9 @@ export const whisperXSpeechAlignmentFragment = sealGraphFragment({
root: operation("request-whisperx"),
},
{
- name: "map",
- type: semanticMapTypes.complete,
- root: operation("locate-speech"),
+ name: "take",
+ type: speechTypes.semanticTake,
+ root: operation("align-semantic-take"),
},
],
});
diff --git a/packages/whisperx/src/index.ts b/packages/whisperx/src/index.ts
index 35a5d0a7..40164f6a 100644
--- a/packages/whisperx/src/index.ts
+++ b/packages/whisperx/src/index.ts
@@ -1,8 +1,8 @@
export {
whisperXComponent,
} from "./component.js";
-export { whisperXSpeechAlignmentFragment } from "./fragment.js";
+export { whisperXSemanticTakeFragment } from "./fragment.js";
export { whisperXRequestForEvidenceAudio } from "./evidence.js";
export { whisperXCapabilities, whisperXManifest, whisperXMarkupSurfaces, whisperXModuleRef, whisperXProducers } from "./manifest.js";
-export { decodeWhisperXAlignmentSurface } from "./surface.js";
+export { decodeWhisperXSemanticTakeSurface } from "./surface.js";
export type * from "./types.js";
diff --git a/packages/whisperx/src/manifest.ts b/packages/whisperx/src/manifest.ts
index abd26228..e7e835ae 100644
--- a/packages/whisperx/src/manifest.ts
+++ b/packages/whisperx/src/manifest.ts
@@ -1,7 +1,7 @@
-import { narrativeTypes } from "@hypit/narrative";
import { speechDependency, speechTypes } from "@hypit/speech";
+import { mediaDependency, mediaTypes } from "@hypit/media";
+import { narrativeTypes } from "@hypit/narrative";
import { speechEvidenceDependency, speechEvidenceTypes } from "@hypit/speech-evidence";
-import { semanticMapDependency, semanticMapTypes } from "@hypit/semantic-map";
import type { CapabilityRef, ModuleManifest, ProducerRef } from "@hypit/protocol";
import { mediaPipelineManifest, mediaPipelineModuleRef } from "@hypit/media-pipeline";
import { speechAlignmentManifest, speechAlignmentModuleRef } from "@hypit/speech-alignment";
@@ -15,30 +15,34 @@ export const whisperXProducers = {
} satisfies Record<string, ProducerRef>;
export const whisperXMarkupSurfaces = [{
- name: "alignment",
- tag: "Alignment",
+ name: "semantic-take",
+ tag: "SemanticTake",
mode: "structured",
- outputs: [speechEvidenceTypes.alignedTranscript, semanticMapTypes.complete],
+ outputs: [speechEvidenceTypes.alignedTranscript, speechTypes.semanticTake],
vocabulary: {
summary:
- "Runs one WhisperX acoustic pass over the speech audio and locates the authored Narrative in it, publishing the aligned transcript and the SemanticMap that carries word timing.",
+ "Measures one normalized Take with WhisperX and aligns one authored Segment into a self-contained SemanticTake.",
attributes: [
{ name: "id", kind: "identifier", required: true,
summary: "Names this alignment and prefixes the bindings it publishes." },
{ name: "narrative", kind: "reference", required: true,
accepts: [narrativeTypes.narrative],
- summary: "Selects the authored Narrative whose Segments the measured speech is assigned to." },
- { name: "audio", kind: "reference", required: true,
- accepts: [speechTypes.audioBasis],
- summary: "Selects the speech audio basis this element measures and aligns against." },
+ summary: "Selects the authored Narrative that owns the Segment and Token identities." },
+ { name: "segment", kind: "reference", required: true,
+ accepts: [narrativeTypes.excerpt],
+ summary: "Selects the single authored Segment performed by this Take." },
+ { name: "media", kind: "reference", required: true,
+ accepts: [mediaTypes.synchronized],
+ summary: "Selects the already normalized SynchronizedMedia measured by WhisperX." },
],
ports: [
{ name: "evidence", type: speechEvidenceTypes.alignedTranscript,
summary: "The provider-neutral aligned transcript WhisperX returned, addressed as `<id>.evidence`." },
- { name: "map", type: semanticMapTypes.complete,
- summary: "The complete SemanticMap placing every authored word in the frame domain, addressed as `<id>.map`." },
+ { name: "take", type: speechTypes.semanticTake,
+ summary: "The normalized media plus this Segment's authored words and local frame anchors." },
],
- example: `<whisperx:Alignment id="timing" narrative={story} audio={speech.audio}/>`,
+ example: `<whisperx:SemanticTake id="opening" narrative={story}
+ segment={story.segment.opening} media={opening-media.media}/>`,
notes: [
"All three attributes are required; the element accepts no children and no text content.",
"Importing this package is what selects the WhisperX model family; the Runtime separately binds the alignment Need to an Endpoint.",
@@ -54,7 +58,7 @@ export const whisperXManifest: ModuleManifest = {
dependencies: [
speechDependency,
speechEvidenceDependency,
- semanticMapDependency,
+ mediaDependency,
{ module: mediaPipelineModuleRef },
{ module: speechAlignmentModuleRef },
],
diff --git a/packages/whisperx/src/surface.ts b/packages/whisperx/src/surface.ts
index f04b8493..e6fdfd66 100644
--- a/packages/whisperx/src/surface.ts
+++ b/packages/whisperx/src/surface.ts
@@ -1,29 +1,16 @@
-import { narrativeTypes } from "@hypit/narrative";
-import { speechTypes } from "@hypit/speech";
-import type {
- StructuredElement,
- StructuredSurfaceHandler,
- SurfaceResolvedReference,
- MarkupAttributeValue,
+import {
+ assertExactAttributes as exactAttributes,
+ textAttribute as stringAttribute,
+ type StructuredElement,
+ type StructuredSurfaceHandler,
+ type SurfaceResolvedReference,
+ type MarkupAttributeValue,
} from "@hypit/markup";
+import { sameType } from "@hypit/protocol";
+import { narrativeTypes } from "@hypit/narrative";
+import { mediaTypes } from "@hypit/media";
-import { whisperXSpeechAlignmentFragment } from "./fragment.js";
-
-function exactAttributes(element: StructuredElement, names: readonly string[]): void {
- if (Object.keys(element.attributes).sort().join("\u0000") !== [...names].sort().join("\u0000")) {
- throw new Error(`${element.name} requires exactly ${names.join(", ")}`);
- }
-}
-
-function stringAttribute(element: StructuredElement, name: string): string {
- const value = element.attributes[name];
- if (typeof value !== "string" || !value.trim()) throw new Error(`${element.name}.${name} must be a non-empty string`);
- return value.trim();
-}
-
-function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
- return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
-}
+import { whisperXSemanticTakeFragment } from "./fragment.js";
function reference(
element: StructuredElement,
@@ -39,26 +26,27 @@ function reference(
return value;
}
-export const decodeWhisperXAlignmentSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- exactAttributes(element, ["id", "narrative", "audio"]);
+export const decodeWhisperXSemanticTakeSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ exactAttributes(element, ["id", "narrative", "segment", "media"]);
if (element.children.some((child) => child.kind === "element" || child.value.trim())) {
throw new Error(`${element.name} does not accept children`);
}
const id = stringAttribute(element, "id");
const narrative = reference(element, "narrative", narrativeTypes.narrative, resolveReference);
- const audio = reference(element, "audio", speechTypes.audioBasis, resolveReference);
+ const segment = reference(element, "segment", narrativeTypes.excerpt, resolveReference);
+ const media = reference(element, "media", mediaTypes.synchronized, resolveReference);
return {
records: [],
components: [{
id,
- fragment: whisperXSpeechAlignmentFragment.id,
- inputs: { narrative: narrative.ref, audio: audio.ref },
+ fragment: whisperXSemanticTakeFragment.id,
+ inputs: { narrative: narrative.ref, segment: segment.ref, media: media.ref },
outputs: {
evidence: `${id}.evidence`,
- map: `${id}.map`,
+ take: `${id}.take`,
},
range: element.range,
}],
- fragments: [whisperXSpeechAlignmentFragment],
+ fragments: [whisperXSemanticTakeFragment],
};
};
diff --git a/packages/whisperx/test/whisperx.test.ts b/packages/whisperx/test/whisperx.test.ts
index 0d337ecd..d5c464f8 100644
--- a/packages/whisperx/test/whisperx.test.ts
+++ b/packages/whisperx/test/whisperx.test.ts
@@ -2,9 +2,8 @@ import {
registerProducerFacets,
} from "@hypit/component-kit";
import type { ProducerRegistrar } from "@hypit/component-kit";
-import { sealProgramSpace } from "@hypit/program-space";
-import { sealSpeechBasis, sealSpeechEvidenceAudio } from "@hypit/speech";
-import type { SpeechBasis, SpeechEvidenceAudio } from "@hypit/speech";
+import { sealSpeechEvidenceAudio } from "@hypit/speech";
+import type { SpeechEvidenceAudio } from "@hypit/speech";
import assert from "node:assert/strict";
import test from "node:test";
import { fixtureDigest } from "../../../test/fixture-digest.js";
@@ -16,20 +15,7 @@ import {
whisperXRequestForEvidenceAudio,
} from "@hypit/whisperx";
-function basis() {
- const programSpace = sealProgramSpace({
- durationSec: 1,
- frameRate: { numerator: 30, denominator: 1 },
- });
- return sealSpeechBasis({
- programSpace,
- audio: { kind: "blob", digest: fixtureDigest("whisperx-test:audio"), size: 1, mediaType: "audio/wav" },
- visualTrack: { clips: [] },
- segments: [{ segmentId: "line", startFrame: 0, endFrameExclusive: 30 }],
- });
-}
-
-function evidenceAudio(basis: SpeechBasis): SpeechEvidenceAudio {
+function evidenceAudio(): SpeechEvidenceAudio {
return sealSpeechEvidenceAudio({
artifact: {
kind: "blob",
@@ -42,8 +28,7 @@ function evidenceAudio(basis: SpeechBasis): SpeechEvidenceAudio {
}
test("WhisperX receives normalized bytes without authored Segment truth", () => {
- const source = basis();
- const valid = evidenceAudio(source);
+ const valid = evidenceAudio();
const request = whisperXRequestForEvidenceAudio(valid);
assert.equal(request.audio.digest, valid.artifact.digest);
assert.equal("segments" in request, false);