Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(speech): carry wordless segments through semantic takes
kashorin61bcc699c561
diff --git a/packages/media-track/README.md b/packages/media-track/README.mdindex fc1a05a6..c8f19573 100644--- a/packages/media-track/README.md+++ b/packages/media-track/README.md@@ -1,11 +1,10 @@# `@hypit/media-track`Place images, prepared video or compositable surfaces over the semantic timeline. B-roll is a common-use: the picture can illustrate a phrase while the A-roll's speech continues. The same Track can-present a speaking A-roll as an inset or a prepared transparent cutout; its SemanticTake continues-to establish semantic time. The choice of visual component does not determine the performance's role.-Items have independent Windows; a Sequence owns one visual slot whose Members replace each other-through Handoffs.+use: the picture can illustrate a phrase while the A-roll's speech continues. A prepared speaking+A-roll is normally presented directly by Speech Track, including an inset or transparent cutout, so+its semantic, visual and audio outputs keep one owner. Media Items own independently timed pictures;+a Sequence owns one visual slot whose Members replace each other through Handoffs.The Markup Track takes `semantic` and `canvas`. Its Surface projects authored Selections, Moments,Segments or clock expressions into Windows and Instants. The component consumes those projecteddiff --git a/packages/media-track/preview/build.svrun b/packages/media-track/preview/build.svrunindex 42e2994f..c1b5bdfc 100644--- a/packages/media-track/preview/build.svrun+++ b/packages/media-track/preview/build.svrun@@ -3,18 +3,19 @@<svrun version="1"><author source="./preview.svml"/><import from="@hypit/stand-in@1" as="stand-in"/>+ <import from="@hypit/semantic-take-estimate@1" as="estimate"/><target output="final.video"/>- <fragment id="take-card" using="stand-in:video">+ <fragment id="take-card" using="stand-in:timed-card"><input name="canvas" from="vertical"/><input name="duration" value="8"/><input name="clock" from="clock"/></fragment>- <fragment id="motion-a-card" using="stand-in:video">+ <fragment id="motion-a-card" using="stand-in:timed-card"><input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/></fragment>- <fragment id="motion-b-card" using="stand-in:video">+ <fragment id="motion-b-card" using="stand-in:timed-card"><input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/>@@ -22,4 +23,11 @@<satisfy output="take.video" candidate="take-card.video"/><satisfy output="motion-a.video" candidate="motion-a-card.video"/><satisfy output="motion-b.video" candidate="motion-b-card.video"/>+ <fragment id="take-timing" using="estimate:semantic-take">+ <input name="narrative" from="story"/>+ <input name="segment" from="story.segment.showcase"/>+ <input name="media" from="take-preview-media.media"/>+ <input name="policy" from="showcase-estimated.policy"/>+ </fragment>+ <satisfy output="showcase-semantic.take" candidate="take-timing.take"/></svrun>diff --git a/packages/media-track/preview/preview.svml b/packages/media-track/preview/preview.svmlindex 6926937b..c6ba09c8 100644--- a/packages/media-track/preview/preview.svml+++ b/packages/media-track/preview/preview.svml@@ -15,7 +15,7 @@in Studio as in a Build:<import from="@hypit/stand-in@1" as="stand-in"/>- <fragment id="a-card" using="stand-in:video">+ <fragment id="a-card" using="stand-in:timed-card"><input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/></fragment><satisfy output="motion-a.video" candidate="a-card.video"/>@@ -29,6 +29,7 @@<import as="pipeline" from="@hypit/media-pipeline@1"/><import as="speech" from="@hypit/speech-track@1"/><import as="whisperx" from="@hypit/whisperx@1"/>+ <import as="estimate" from="@hypit/semantic-take-estimate@1"/><import as="media-track" from="@hypit/media-track@1"/><import as="space" from="@hypit/spatial@1"/><import as="program" from="@hypit/program-space@1"/>@@ -59,12 +60,17 @@<pipeline:Normalize id="take-media" source={take.video}recipe={recipes.media.aroll} clock={clock}/>+ <pipeline:Normalize id="take-preview-media" source={take.video} clock={clock}+ video="primary-moving" audio="none" span-authority="video"/><pipeline:Normalize id="motion-a-media" source={motion-a.video}recipe={recipes.media.broll} clock={clock}/><pipeline:Normalize id="motion-b-media" source={motion-b.video}recipe={recipes.media.broll} clock={clock}/><whisperx:SemanticTake id="showcase-semantic" narrative={story}segment={story.segment.showcase} media={take-media.media} language="en"/>+ <estimate:SemanticTake id="showcase-estimated" narrative={story}+ segment={story.segment.showcase} media={take-preview-media.media}+ language="en" pace="normal" rounding="round"/><speech:Track id="speech"visual-frame={speech-frame} visual-appearance={recipes.speech.visual} visual-z="0">diff --git a/packages/media-track/src/author.ts b/packages/media-track/src/author.tsindex 5bacde1b..72b16404 100644--- a/packages/media-track/src/author.ts+++ b/packages/media-track/src/author.ts@@ -16,6 +16,7 @@ import type {MediaPaint,MediaPaintLayerSpec,MediaSampleLayerSpec,+ MediaSampleAppearance,MediaSequenceSpec,MediaVisualOccupancy,MediaVisualTrim,@@ -175,17 +176,22 @@ export function decodeMediaSampleSpec(id,...(timed ? { occupancy: occupancy(recipe), ...(sourceTrim === undefined ? {} : { trim: sourceTrim }) } : {}),- appearance: {- opacity: number(recipe, "opacity", 1),- filter: {- blurPx: number(recipe, "blur", 0), brightness: number(recipe, "brightness", 1),- contrast: number(recipe, "contrast", 1), saturation: number(recipe, "saturation", 1),- },- },+ appearance: decodeMediaSampleAppearance(recipe),...(samplingMotion === undefined ? {} : { samplingMotion }),});}+/** Decode visual-only sample styling without choosing playback or source trim. */+export function decodeMediaSampleAppearance(recipe: SvsRecipe): MediaSampleAppearance {+ return {+ opacity: number(recipe, "opacity", 1),+ filter: {+ blurPx: number(recipe, "blur", 0), brightness: number(recipe, "brightness", 1),+ contrast: number(recipe, "contrast", 1), saturation: number(recipe, "saturation", 1),+ },+ };+}+function edge(recipe: SvsRecipe, prefix: "enter" | "exit"): MediaLifecycleMotion["enter"] {const operator = optionalString(recipe, prefix);if (operator === undefined || operator === "none") return undefined;diff --git a/packages/media-track/src/index.ts b/packages/media-track/src/index.tsindex 1ebb751d..9bc00617 100644--- a/packages/media-track/src/index.ts+++ b/packages/media-track/src/index.ts@@ -4,6 +4,7 @@ export {decodeMediaFramePaint,decodeMediaMotion,decodeMediaPresentation,+ decodeMediaSampleAppearance,decodeMediaSampleSpec,mediaAppearanceKeys,} from "./author.js";@@ -23,7 +24,7 @@ export {sealMediaSampleLayerSpec,} from "./layers.js";export { lowerMediaItemElements } from "./lower.js";-export { mediaItemSpecSchema, mediaFramePresentationSchema, mediaLifecycleMotionSchema, mediaLayerSetSchema, mediaHandoffSpecSchema, mediaPaintLayerSpecSchema, mediaSampleLayerSpecSchema, mediaSoundSetSchema, mediaSoundSpecSchema, mediaSequenceMemberSetSchema, mediaSequenceMemberSpecSchema, mediaSequenceSpecSchema, mediaTrackDependency, mediaTrackHeaderSchema, mediaTrackManifest, mediaTrackMarkupSurfaces, mediaTrackModuleRef, mediaTrackProducers, mediaTrackProgramSchema, mediaTrackSetSchema, mediaTrackTypes } from "./manifest.js";+export { mediaItemSpecSchema, mediaFramePresentationSchema, mediaLifecycleMotionSchema, mediaLayerSetSchema, mediaHandoffSpecSchema, mediaPaintLayerSpecSchema, mediaSampleAppearanceSchema, mediaSampleLayerSpecSchema, mediaSamplingMotionSchema, mediaSoundSetSchema, mediaSoundSpecSchema, mediaSequenceMemberSetSchema, mediaSequenceMemberSpecSchema, mediaSequenceSpecSchema, mediaTrackDependency, mediaTrackHeaderSchema, mediaTrackManifest, mediaTrackMarkupSurfaces, mediaTrackModuleRef, mediaTrackProducers, mediaTrackProgramSchema, mediaTrackSetSchema, mediaTrackTypes } from "./manifest.js";export {assertMediaEdgeMotion,assertMediaLifecycleMotion,@@ -47,7 +48,7 @@ export { appendMediaItem, appendMediaSequence, bindMediaItemClipPath, bindMediaSexport { resolveVisualSampling } from "./sampling.js";export { lowerRestrictedSpeechVisualPresents } from "./restricted-speech.js";export type { RestrictedSpeechVisualClip } from "./restricted-speech.js";-export { decodeMediaTrackSurface } from "./surface.js";+export { decodeMediaSamplingKeyframe, decodeMediaTrackSurface } from "./surface.js";export {appendMediaSequenceMember,assertMediaHandoffSpec,diff --git a/packages/media-track/src/manifest.ts b/packages/media-track/src/manifest.tsindex 76791c5e..48a8b8da 100644--- a/packages/media-track/src/manifest.ts+++ b/packages/media-track/src/manifest.ts@@ -94,7 +94,7 @@ const paint: ValueSchema = { kind: "oneOf", variants: [object({ kind: { schema: { kind: "literal", value: "linear-gradient" } }, angleDeg: { schema: number }, stops: { schema: stops } }),object({ kind: { schema: { kind: "literal", value: "radial-gradient" } }, center: { schema: object({ x: { schema: unsigned }, y: { schema: unsigned } }) }, stops: { schema: stops } }),] };-const appearance = object({+export const mediaSampleAppearanceSchema: ValueSchema = object({opacity: { schema: unsigned },filter: { schema: object({blurPx: { schema: unsigned }, brightness: { schema: unsigned }, contrast: { schema: unsigned }, saturation: { schema: unsigned },@@ -105,7 +105,9 @@ const samplingKeyframe = object({offsetX: { schema: number }, offsetY: { schema: number }, rotationDeg: { schema: number },easing: { schema: { kind: "string", enum: ["linear", "ease-in", "ease-out", "ease-in-out"] }, optional: true },});-const samplingMotion = object({ keyframes: { schema: { kind: "array", minItems: 2, items: samplingKeyframe } } });+export const mediaSamplingMotionSchema: ValueSchema = object({+ keyframes: { schema: { kind: "array", minItems: 2, items: samplingKeyframe } },+});export const mediaPaintLayerSpecSchema: ValueSchema = object({id: { schema: string }, paint: { schema: paint }, opacity: { schema: unsigned },@@ -113,7 +115,7 @@ export const mediaPaintLayerSpecSchema: ValueSchema = object({export const mediaSampleLayerSpecSchema: ValueSchema = object({id: { schema: string }, trim: { schema: trim, optional: true }, occupancy: { schema: occupancy, optional: true },- appearance: { schema: appearance }, samplingMotion: { schema: samplingMotion, optional: true },+ appearance: { schema: mediaSampleAppearanceSchema }, samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },});const audioSource = object({ artifact: { schema: blob }, sampleFrames: { schema: positiveInteger } });const visualSource: ValueSchema = { kind: "oneOf", variants: [@@ -128,7 +130,7 @@ const paintLayer = object({ id: { schema: string }, kind: { schema: { kind: "litconst sampleLayer = object({id: { schema: string }, kind: { schema: { kind: "literal", value: "sample" } }, source: { schema: visualSource },fit: { schema: contentFitSchema }, trim: { schema: trim, optional: true }, occupancy: { schema: occupancy, optional: true },- appearance: { schema: appearance }, samplingMotion: { schema: samplingMotion, optional: true },+ appearance: { schema: mediaSampleAppearanceSchema }, samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },});const layer = { kind: "oneOf", variants: [paintLayer, sampleLayer] } as const;export const mediaLayerSetSchema: ValueSchema = object({diff --git a/packages/media-track/src/restricted-speech.ts b/packages/media-track/src/restricted-speech.tsindex 3d650fb9..f64cc7aa 100644--- a/packages/media-track/src/restricted-speech.ts+++ b/packages/media-track/src/restricted-speech.ts@@ -6,7 +6,14 @@ import { assertContentFit, assertIntrinsicExtent, assertSpatialFrame } from "@hyimport type { ContentFit, IntrinsicExtent, SpatialFrame } from "@hypit/spatial";import { lowerMediaItemElements } from "./lower.js";-import type { MediaItemProgram } from "./types.js";+import type {+ MediaFramePresentation,+ MediaItemProgram,+ MediaLifecycleMotion,+ MediaPaintLayerSpec,+ MediaSampleAppearance,+ MediaSamplingMotion,+} from "./types.js";export type RestrictedSpeechVisualClip = {readonly id: string;@@ -19,12 +26,17 @@ export type RestrictedSpeechVisualClip = {readonly frame: SpatialFrame;readonly fit: ContentFit;readonly stackingOrder: number;+ readonly presentation: MediaFramePresentation;+ readonly sampleAppearance: MediaSampleAppearance;+ readonly motion: MediaLifecycleMotion;+ readonly framePaint?: MediaPaintLayerSpec;+ readonly samplingMotion?: MediaSamplingMotion;};/**- * Focused reuse point for Speech Track: one normalized muted take and one- * explicitly placed foreground layer, without Media Track motion or sequence- * semantics.+ * Focused reuse point for Speech Track: one normalized muted Take and its+ * visual presentation inside a fixed Segment envelope, without Media Track+ * Window, playback/trim, source-audio or Sequence ownership.*/export function lowerRestrictedSpeechVisualPresents(trackId: string,@@ -57,29 +69,33 @@ export function lowerRestrictedSpeechVisualPresents(subjectId: clip.subjectId ?? clip.id,span: { ...clip.span },frame: structuredClone(clip.frame),- presentation: {- clip: { kind: "frame" },- padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 },- shadows: [],- },- layers: [{- id: `${trackId}:${clip.id}:foreground`,- kind: "sample",- source: {- kind: "timed",- artifact: structuredClone(clip.artifact),- extent: structuredClone(clip.extent),- frameRate: { ...clip.frameRate },- frameCount: clip.frameCount,- },- fit: structuredClone(clip.fit),- occupancy: { mode: "once", align: "start" },- appearance: {- opacity: 1,- filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 },+ presentation: structuredClone(clip.presentation),+ layers: [+ ...(clip.framePaint === undefined ? [] : [{+ id: clip.framePaint.id,+ kind: "paint" as const,+ paint: structuredClone(clip.framePaint.paint),+ opacity: clip.framePaint.opacity,+ }]),+ {+ id: `${trackId}:${clip.id}:foreground`,+ kind: "sample",+ source: {+ kind: "timed",+ artifact: structuredClone(clip.artifact),+ extent: structuredClone(clip.extent),+ frameRate: { ...clip.frameRate },+ frameCount: clip.frameCount,+ },+ fit: structuredClone(clip.fit),+ occupancy: { mode: "once", align: "start" },+ appearance: structuredClone(clip.sampleAppearance),+ ...(clip.samplingMotion === undefined ? {} : {+ samplingMotion: structuredClone(clip.samplingMotion),+ }),},- }],- motion: { sustain: [] },+ ],+ motion: structuredClone(clip.motion),stacking: {order: clip.stackingOrder,tieBreak: `${trackId}:${String(index + 1).padStart(4, "0")}:${clip.id}`,diff --git a/packages/media-track/src/surface.ts b/packages/media-track/src/surface.tsindex 6b9a33a7..c8053241 100644--- a/packages/media-track/src/surface.ts+++ b/packages/media-track/src/surface.ts@@ -307,7 +307,7 @@ function declaredVisualSource(: { kind, value: reference(element.attributes.surface, `${element.name}.surface`, mediaTypes.compositableSurface, resolve) };}-function sampling(element: StructuredElement): MediaSamplingMotion["keyframes"][number] {+export function decodeMediaSamplingKeyframe(element: StructuredElement): MediaSamplingMotion["keyframes"][number] {allowed(element, ["at", "zoom", "x", "y", "rotate", "easing"]);empty(element);const at = text(element, "at");@@ -348,7 +348,7 @@ function sourceLayer(if (context.ignoreSiblingChildren === true) return [];throw new Error(`${element.name} accepts only Sampling children.`);}- return [sampling(child)];+ return [decodeMediaSamplingKeyframe(child)];});const motion = keyframes.length === 0 ? undefined : { keyframes };const layerSuffix = `${context.unitSuffix}-layer-${suffix(layerIndex)}`;diff --git a/packages/narrative/src/identity.ts b/packages/narrative/src/identity.tsindex 583ef29a..1af8abc2 100644--- a/packages/narrative/src/identity.ts+++ b/packages/narrative/src/identity.ts@@ -38,7 +38,7 @@ export function assertNarrativeIdentity(value: Narrative): void {let tokenCursor = 0;for (const segment of value.segments) {if (!Number.isSafeInteger(segment.tokenStart) || !Number.isSafeInteger(segment.tokenEndExclusive)- || segment.tokenStart !== tokenCursor || segment.tokenEndExclusive <= segment.tokenStart+ || segment.tokenStart !== tokenCursor || segment.tokenEndExclusive < segment.tokenStart|| segment.tokenEndExclusive > value.tokens.length|| !anchorIds.has(segment.startAnchorId) || !anchorIds.has(segment.endAnchorId)) {throw new Error(`Narrative Segment ${segment.id} has invalid token or Anchor boundaries.`);@@ -79,7 +79,7 @@ export function assertNarrativeExcerptIdentity(value: NarrativeExcerpt): void {nonempty(value.narrativeId, "NarrativeExcerpt narrativeId");nonempty(value.id, "NarrativeExcerpt id");if (!Number.isSafeInteger(value.tokenStart) || !Number.isSafeInteger(value.tokenEndExclusive)- || value.tokenStart < 0 || value.tokenEndExclusive <= value.tokenStart) {+ || value.tokenStart < 0 || value.tokenEndExclusive < value.tokenStart) {throw new Error("NarrativeExcerpt Token coverage is invalid.");}}@@ -100,7 +100,9 @@ export function assertNarrativeMomentRefIdentity(value: NarrativeMomentRef): voiexport function assertCaptionDocumentIdentity(value: CaptionDocument): void {nonempty(value.narrativeId, "CaptionDocument narrativeId");nonempty(value.id, "CaptionDocument id");- if (value.units.length === 0 || value.words.length === 0) throw new Error("CaptionDocument is empty.");+ if ((value.units.length === 0) !== (value.words.length === 0)) {+ throw new Error("CaptionDocument units and words must be empty together.");+ }const unitIds = unique(value.units.map((item) => item.id), "CaptionDocument unit id");const wordIds = unique(value.words.map((item) => item.id), "CaptionDocument word id");const words = new Map(value.words.map((word) => [word.id, word] as const));diff --git a/packages/narrative/src/schema.ts b/packages/narrative/src/schema.tsindex b86f8b14..534b545b 100644--- a/packages/narrative/src/schema.ts+++ b/packages/narrative/src/schema.ts@@ -73,8 +73,8 @@ const captionCueBreak = object({ afterUnitId: { schema: string } });export const captionDocumentSchema: ValueSchema = object({narrativeId: { schema: string },id: { schema: string },- units: { schema: { kind: "array", minItems: 1, items: captionAlignmentUnit } },- words: { schema: { kind: "array", minItems: 1, items: captionDisplayWord } },+ units: { schema: { kind: "array", items: captionAlignmentUnit } },+ words: { schema: { kind: "array", items: captionDisplayWord } },cueBreaks: { schema: { kind: "array", items: captionCueBreak } },});export const narrativeMomentSchema: ValueSchema = object({diff --git a/packages/provider-hyperframes-local/test/alpha.test.ts b/packages/provider-hyperframes-local/test/alpha.test.tsindex 765c28b0..5ddcc98a 100644--- a/packages/provider-hyperframes-local/test/alpha.test.ts+++ b/packages/provider-hyperframes-local/test/alpha.test.ts@@ -44,7 +44,12 @@ test("transparent normalized media composites through both SemanticTake/Speech Tconst fit = { sizing: "contain" as const, framePoint: { x: 0.5, y: 0.5 }, contentPoint: { x: 0.5, y: 0.5 },offsetPx: { x: 0, y: 0 }, constraint: "bounded" as const };const frame = { xPx: 0, yPx: 0, widthPx: 96, heightPx: 64 };- const set = appendSpeechTrackTake(createSpeechTrackSet(), take, frame, fit, sealSpeechTrackVisualSpec({ stackingOrder: 1 }));+ const set = appendSpeechTrackTake(createSpeechTrackSet(), take, frame, fit, sealSpeechTrackVisualSpec({+ stackingOrder: 1,+ presentation: { clip: { kind: "frame" }, padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 }, shadows: [] },+ sampleAppearance: { opacity: 1, filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 } },+ motion: { sustain: [] },+ }));const semantic = assembleSpeechTrack(sealSpeechTrackHeader({ id: "speech" }), set);const speechVisual = projectSpeechTrackVisual(semantic, set);const space = projectSemanticProgramSpace(semantic);diff --git a/packages/semantic-take-estimate/README.md b/packages/semantic-take-estimate/README.mdindex faad4958..354ddc41 100644--- a/packages/semantic-take-estimate/README.md+++ b/packages/semantic-take-estimate/README.md@@ -71,5 +71,5 @@ the `opening-estimated` declaration above, a Run can use:This selects estimated timing for the measured Take while retaining the required prepared-mediadependency and any Run choice for its source bytes. The estimate allocates the media's complete-frame count; it neither generates nor normalizes that media. It requires spoken Tokens. A wordless-Segment uses actual media boundaries directly when materializing its SemanticTake.+frame count; it neither generates nor normalizes that media. For a wordless Segment it maps the+Segment's two boundary Anchors directly to the media's first and final frame.diff --git a/packages/semantic-take-estimate/src/manifest.ts b/packages/semantic-take-estimate/src/manifest.tsindex d73642f5..cfce1421 100644--- a/packages/semantic-take-estimate/src/manifest.ts+++ b/packages/semantic-take-estimate/src/manifest.ts@@ -18,7 +18,7 @@ export const semanticTakeEstimateMarkupSurfaces = [{outputs: [estimateTypes.speechPolicy, speechTypes.semanticTake],vocabulary: {summary:- "Projects syllable-weighted estimated word windows onto one already normalized video and publishes a SemanticTake.",+ "Projects estimated word windows or wordless Segment boundaries onto normalized media and publishes a SemanticTake.",attributes: [{ name: "id", kind: "identifier", required: true,summary: "Names this estimated alignment and the SemanticTake it publishes." },@@ -57,7 +57,7 @@ export const semanticTakeEstimateMarkupSurfaces = [{notes: ["The policy is the element's own: write it inline or name an SVS Recipe with policy; never both.","This package does not measure audio and never claims WhisperX or other acoustic evidence.",- "Every Token uses its original Script identity. Only its local frame window is estimated.",+ "Every spoken Token uses its original Script identity. A wordless Segment keeps its two authored boundary Anchors.","Pronunciation-unit weights determine relative word lengths; adjacent words and both media edges retain non-zero frame gaps.","The result has the same SemanticTake Type as a measured alignment, so Run chooses between ordinary candidates without a preview-specific downstream protocol.",],diff --git a/packages/semantic-take-estimate/src/program.ts b/packages/semantic-take-estimate/src/program.tsindex 3afc64e7..36e3d0ad 100644--- a/packages/semantic-take-estimate/src/program.ts+++ b/packages/semantic-take-estimate/src/program.ts@@ -22,7 +22,6 @@ function segmentTokens(narrative: Narrative, excerpt: NarrativeExcerpt): readonlthrow new Error(`NarrativeExcerpt ${excerpt.id} does not describe its authored Segment.`);}const tokens = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);- if (tokens.length === 0) throw new Error(`Estimated SemanticTake Segment ${excerpt.id} contains no speech Tokens.`);return tokens;}@@ -49,6 +48,21 @@ export function estimateSemanticTakeTiming(verifySynchronizedMedia(media);assertSpeechEstimatePolicy(policy);const tokens = segmentTokens(narrative, excerpt);+ const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id)!;+ if (tokens.length === 0) {+ const frameByAnchor = new Map<string, number>([+ [segment.startAnchorId, 0],+ [segment.endAnchorId, media.timeline.frameCount],+ ]);+ const anchors = narrative.semanticIndex.anchors+ .filter((anchor) => anchor.segmentId === excerpt.id)+ .map((anchor) => {+ const frame = frameByAnchor.get(anchor.id);+ if (frame === undefined) throw new Error(`Estimated timing cannot locate authored Anchor ${anchor.id}.`);+ return { identity: anchor.id, frame };+ });+ return { tokens: [], anchors };+ }const rate = media.timeline.frameRate.numerator / media.timeline.frameRate.denominator;const edgeGap = Math.max(1, Math.round(EDGE_GAP_SECONDS * rate));const wordGap = Math.max(1, Math.round(WORD_GAP_SECONDS * rate));@@ -79,7 +93,6 @@ export function estimateSemanticTakeTiming(throw new Error("Estimated SemanticTake timing allocation did not close over its frame domain.");}- const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id)!;const frameByAnchor = new Map<string, number>([[segment.startAnchorId, 0],[segment.endAnchorId, media.timeline.frameCount],diff --git a/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts b/packages/semantic-take-estimate/test/semantic-take-estimate.test.tsindex 431385b3..d72a9f70 100644--- a/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts+++ b/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts@@ -3,6 +3,11 @@ import test from "node:test";import { sealSpeechEstimatePolicy } from "@hypit/estimate";import { sealSynchronizedMedia } from "@hypit/media";+import {+ assertCaptionDocumentIdentity,+ assertNarrativeExcerptIdentity,+ assertNarrativeIdentity,+} from "@hypit/narrative";import type { Narrative } from "@hypit/narrative";import { fixtureResource } from "../../../test/fixture-resource.js";@@ -106,3 +111,50 @@ test("an implausibly short frame domain is refused instead of squeezing words to/keep every Token visible and separated/u,);});++test("a wordless Segment receives the prepared media boundaries without an audio substitute", () => {+ const wordless: Narrative = {+ id: "wordless",+ segments: [{+ id: "pause",+ startAnchorId: "pause:start",+ endAnchorId: "pause:end",+ tokenStart: 0,+ tokenEndExclusive: 0,+ }],+ tokens: [],+ turns: [],+ selections: [],+ moments: [],+ semanticIndex: { anchors: [+ { id: "program:start", kind: "program-start" },+ { id: "pause:start", kind: "segment-start", segmentId: "pause" },+ { id: "pause:end", kind: "segment-end", segmentId: "pause" },+ { id: "program:end", kind: "program-end" },+ ] },+ };+ const wordlessExcerpt = {+ narrativeId: wordless.id,+ kind: "segment" as const,+ id: "pause",+ tokenStart: 0,+ tokenEndExclusive: 0,+ };+ assertNarrativeIdentity(wordless);+ assertNarrativeExcerptIdentity(wordlessExcerpt);+ assertCaptionDocumentIdentity({+ narrativeId: wordless.id,+ id: "wordless.caption",+ units: [],+ words: [],+ cueBreaks: [],+ });++ const take = materializeEstimatedSemanticTake(wordless, wordlessExcerpt, media, policy);+ assert.deepEqual(take.tokens, []);+ assert.deepEqual(take.anchors, [+ { identity: "pause:start", frame: 0 },+ { identity: "pause:end", frame: media.timeline.frameCount },+ ]);+ assert.equal(take.media.audio, undefined);+});diff --git a/packages/speech-track/README.md b/packages/speech-track/README.mdindex c88ed32f..e9075c28 100644--- a/packages/speech-track/README.md+++ b/packages/speech-track/README.md@@ -7,8 +7,9 @@ speaking turns share this timeline; an independent narration can also carry it.A-roll can be a circular picture-in-picture or a foreground presenter cutout over another mainpicture. Its speaking performance establishes the semantic timeline regardless of its screen area-or stack order. Speech Track supplies that timeline and sound even when Media Track or a project-component supplies the chosen presentation of the same person.+or stack order. Speech Track projects that same prepared Take into semantic, visual and audio+outputs together; the inset or cutout does not need to be displayed a second time through Media+Track.## Assemble prepared Takes@@ -26,9 +27,22 @@ contain several speakers or camera cuts. Takes are assembled in Source order:</speech:Track>```-This excerpt assumes the Takes, Frame and appearance Recipe exist. A fit Recipe can be-`performance { fit: cover; }`. The shared frame rate and total ProgramSpace come from the ordered-Takes. Their local word and boundary frames become positions in that assembled space.+This excerpt assumes the Takes, Frame and appearance Recipe exist. An appearance Recipe can share+the ordinary visual appearance of a Media Item:++```svs+performance {+ fit: cover;+ clip: rounded;+ radius: 270;+ border-width: 3;+ border-color: rgba(255, 255, 255, 0.9);+}+```++With a `540 × 540` Frame, that rounded clip presents the performance as a circle. The shared frame+rate and total ProgramSpace come from the ordered Takes. Their local word and boundary frames become+positions in that assembled space.| Output | Use || --- | --- |@@ -36,16 +50,36 @@ Takes. Their local word and boundary frames become positions in that assembled s| `.visual` | Include the performance picture in Film when wanted || `.audio` | Include the performance sound in Film, including beneath B-roll |+Each Take always contributes its Segment and Anchor timing. A Take with no audio stream contributes+no audio clip; a Take with no visual stream contributes no picture. Its remaining semantic and+media projections continue normally, so a video-only wordless Take needs no synthetic audio and an+audio-only narration needs no synthetic picture.+Select each wanted picture and sound output explicitly in Film. Replacing the visible picture withMedia Track coverage does not replace the semantic timeline or mute the included performance audio.An audio-only performance can supply the timeline while other Tracks supply the picture.-## Place the performance and extend its presentation+## Present the performance directly++`visual-frame`, `visual-appearance`, `visual-motion` and `visual-z` supply the Track's visual base. A+Take can override them with `frame`, `appearance`, `motion` and `z`. Appearance can control fit and+crop anchors, opacity and filters, rounded or rectangular clipping, padding, borders, shadows and+frame paint. Lifecycle motion can move that visual presentation as it enters, remains or exits. None+of these choices changes the Take's semantic timing or audio.++Sampling children move the Take's picture inside its Frame over normalized Segment progress, using+the same fields as a direct Media Item:++```svml+<speech:Take source={opening.take}>+ <speech:Sampling at="start" zoom="1"/>+ <speech:Sampling at="end" zoom="1.08" y="-18" easing="ease-out"/>+</speech:Take>+```-`visual-frame`, `visual-appearance` and `visual-z` supply the Track's visual base. A Take can override-it with `frame`, `appearance` and `z`. This places and stacks the same performance picture. Use-[Media Track](../media-track/README.md) for an independently animated presentation, replacement-Sequence or transition. Caption and MG consume `.semantic` to follow the same words and events.+Use [Media Track](../media-track/README.md) when a picture owns an independent semantic Window,+source playback or trim, source audio, a replacement Sequence, or a visual source other than the+performance itself. Caption and MG consume `.semantic` to follow the same words and events.The [Studio Companion](../speech-track-studio/src/index.ts) presents the assembled Takes and theirmaterials. Media preparation is owned by [Media Pipeline](../media-pipeline/README.md), and exactdiff --git a/packages/speech-track/src/manifest.ts b/packages/speech-track/src/manifest.tsindex bc36425b..77e04c77 100644--- a/packages/speech-track/src/manifest.ts+++ b/packages/speech-track/src/manifest.ts@@ -2,6 +2,13 @@ import { readFile } from "node:fs/promises";import { semanticTakeSchema, speechDependency, speechTypes } from "@hypit/speech";import { compositionDependency, compositionTypes } from "@hypit/composition";+import {+ mediaFramePresentationSchema,+ mediaLifecycleMotionSchema,+ mediaPaintLayerSpecSchema,+ mediaSampleAppearanceSchema,+ mediaSamplingMotionSchema,+} from "@hypit/media-track";import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/protocol";import { semanticTrackDependency, semanticTrackTypes } from "@hypit/semantic-track";import {@@ -41,21 +48,26 @@ export const speechTrackHeaderSchema: ValueSchema = object({id: { schema: string },});+export const speechTrackVisualSpecSchema: ValueSchema = object({+ stackingOrder: { schema: { kind: "number", integer: true } },+ presentation: { schema: mediaFramePresentationSchema },+ sampleAppearance: { schema: mediaSampleAppearanceSchema },+ motion: { schema: mediaLifecycleMotionSchema },+ framePaint: { schema: mediaPaintLayerSpecSchema, optional: true },+ samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },+});+export const speechTrackSetSchema: ValueSchema = object({takes: { schema: { kind: "array", items: object({semantic: { schema: semanticTakeSchema },visual: { schema: object({frame: { schema: spatialFrameSchema },fit: { schema: contentFitSchema },- stackingOrder: { schema: { kind: "number", integer: true } },+ spec: { schema: speechTrackVisualSpecSchema },}), optional: true },}) } },});-export const speechTrackVisualSpecSchema: ValueSchema = object({- stackingOrder: { schema: { kind: "number", integer: true } },-});-const visualRecipeProperties = [{ name: "fit", required: false, fallback: "contain",values: ["contain", "cover", "fit-width", "fit-height", "native", "scale-down", "stretch"],@@ -74,6 +86,59 @@ const visualRecipeProperties = [summary: "Shifts the placed picture vertically in pixels once the two anchor points meet." },{ name: "fit-constraint", required: false, fallback: "bounded", values: ["bounded", "free"],summary: "Decides whether the placed picture is pulled back until it covers as much of the Frame as its size allows, or left exactly where the anchors and offsets put it." },+ { name: "opacity", required: false, fallback: "1",+ summary: "Sets how opaque the Take's picture is drawn." },+ { name: "blur", required: false, fallback: "0",+ summary: "Blurs the Take's picture by a pixel radius." },+ { name: "brightness", required: false, fallback: "1",+ summary: "Scales the brightness of the Take's picture." },+ { name: "contrast", required: false, fallback: "1",+ summary: "Scales the contrast of the Take's picture." },+ { name: "saturation", required: false, fallback: "1",+ summary: "Scales the saturation of the Take's picture." },+ { name: "clip", required: false, values: ["none", "frame", "rounded"], fallback: "frame",+ summary: "Clips the Take's picture to its Frame, leaves overflow visible, or rounds the Frame." },+ { name: "radius", required: false, fallback: "0",+ summary: "Sets the rounded clip radius in pixels; half the side of a square Frame makes a circle." },+ { name: "padding", required: false, fallback: "0",+ summary: "Insets the Take's picture from its Frame edges using one, two or four pixel values." },+ { name: "border-width", required: false, fallback: "0",+ summary: "Draws a border of this pixel width around the Frame." },+ { name: "border-style", required: false, values: ["solid", "dashed", "dotted"], fallback: "solid",+ summary: "Chooses how a non-zero border is stroked." },+ { name: "border-color", required: false,+ summary: "Sets the color of a non-zero border." },+ { name: "shadows", required: false, fallback: "none",+ summary: "Casts Frame shadows written as `x y blur spread color` entries separated by semicolons." },+ { name: "frame-paint", required: false, fallback: "transparent",+ summary: "Fills the Frame behind the Take with a solid or gradient paint." },+] as const;++const visualMotionRecipeProperties = [+ { name: "enter", required: false, fallback: "none",+ values: ["none", "fade", "slide", "scale", "pop", "bounce", "blur-reveal", "wipe", "flip", "spin"],+ summary: "Chooses the visual entrance applied inside this Segment." },+ { name: "enter-frames", required: false,+ summary: "Sets the entrance length in frames when an entrance is selected." },+ { name: "enter-easing", required: false, values: ["linear", "ease-in", "ease-out", "ease-in-out"], fallback: "ease-in-out",+ summary: "Shapes the entrance's acceleration." },+ { name: "enter-direction", required: false, values: ["left", "right", "up", "down"],+ summary: "Sets the direction of a directional entrance." },+ { name: "enter-amount", required: false,+ summary: "Sets how far the entrance displaces the visual." },+ { name: "sustain", required: false, fallback: "none",+ summary: "Applies continuous visual motion over the Segment as `operator amount cycles [direction]`." },+ { name: "exit", required: false, fallback: "none",+ values: ["none", "fade", "slide", "scale", "pop", "bounce", "blur-reveal", "wipe", "flip", "spin"],+ summary: "Chooses the visual exit applied inside this Segment." },+ { name: "exit-frames", required: false,+ summary: "Sets the exit length in frames when an exit is selected." },+ { name: "exit-easing", required: false, values: ["linear", "ease-in", "ease-out", "ease-in-out"], fallback: "ease-in-out",+ summary: "Shapes the exit's acceleration." },+ { name: "exit-direction", required: false, values: ["left", "right", "up", "down"],+ summary: "Sets the direction of a directional exit." },+ { name: "exit-amount", required: false,+ summary: "Sets how far the exit displaces the visual." },] as const;export const speechTrackMarkupSurfaces = [{@@ -97,10 +162,13 @@ export const speechTrackMarkupSurfaces = [{recipe: visualRecipeProperties },{ name: "visual-z", kind: "literal", required: true,summary: "Sets the stacking order every visual Take is composited at unless the Take names its own." },+ { name: "visual-motion", kind: "reference", required: false, accepts: [svsRecipeType],+ summary: "Chooses visual-only entrance, sustained motion and exit for every Take unless a Take names its own.",+ recipe: visualMotionRecipeProperties },],children: [{ tag: "Take", cardinality: "many",- summary: "One self-contained SemanticTake, in document order.",+ summary: "One self-contained SemanticTake, in document order; optional Sampling children move its picture inside the Frame.",attributes: [{ name: "source", kind: "reference", required: true, accepts: [speechTypes.semanticTake],summary: "Chooses the normalized and locally aligned SemanticTake this Track assembles." },@@ -111,6 +179,9 @@ export const speechTrackMarkupSurfaces = [{recipe: visualRecipeProperties },{ name: "z", kind: "literal", required: false,summary: "Sets this Take's own stacking order in place of the Track's `visual-z`." },+ { name: "motion", kind: "reference", required: false, accepts: [svsRecipeType],+ summary: "Chooses this Take's own visual-only motion in place of the Track's `visual-motion`.",+ recipe: visualMotionRecipeProperties },] },],ports: [@@ -128,9 +199,11 @@ export const speechTrackMarkupSurfaces = [{</speech:Track>`,notes: ["A Track requires at least one Take and accepts no text content.",- "`visual-appearance` and a Take's `appearance` must each be an authored SVS Recipe declaring only the spatial fit properties listed for them; any other property is refused.",+ "`visual-appearance` and a Take's `appearance` use the shared Media Item styling listed here: fit, visual filtering, Frame paint, clipping, border and shadow.","Media normalization, acoustic evidence and Segment alignment happen before a Take enters the Track.",- "Placement and stacking are the complete visual authority of a Track; motion, transitions and independent pictures remain ordinary Media Tracks.",+ "Speech visual motion changes only the projected picture inside its Segment; it does not retime the SemanticTake or its audio.",+ "A Take accepts Sampling children with the same `at`, `zoom`, `x`, `y`, `rotate` and `easing` fields as a direct Media Item; at least two keyframes cover normalized Segment progress.",+ "Playback, source trim, independent Windows, replacement Sequences and separate pictures remain ordinary Media Track concerns.",],},}] as const;diff --git a/packages/speech-track/src/program.ts b/packages/speech-track/src/program.tsindex 1f624e91..0c8a0e4d 100644--- a/packages/speech-track/src/program.ts+++ b/packages/speech-track/src/program.ts@@ -3,6 +3,12 @@ import type { SemanticTake } from "@hypit/speech";import { sealSemanticTrack } from "@hypit/semantic-track";import type { SemanticTrack } from "@hypit/semantic-track";import { canonicalize } from "@hypit/protocol";+import {+ assertMediaFramePresentation,+ assertMediaLifecycleMotion,+ assertMediaPaintLayerSpec,+ sealMediaSampleLayerSpec,+} from "@hypit/media-track";import { assertContentFit, assertSpatialFrame } from "@hypit/spatial";import type { ContentFit, SpatialFrame } from "@hypit/spatial";@@ -47,8 +53,7 @@ function assertTake(take: SpeechTrackTake): void {if (take.visual !== undefined) {assertSpatialFrame(take.visual.frame);assertContentFit(take.visual.fit);- assert(Number.isSafeInteger(take.visual.stackingOrder),- `Speech Segment ${segment.segmentId} visual stacking order is invalid`);+ assertSpeechTrackVisualSpec(take.visual.spec);}}@@ -60,6 +65,14 @@ export function sealSpeechTrackVisualSpec(value: SpeechTrackVisualSpec): SpeechTexport function assertSpeechTrackVisualSpec(value: SpeechTrackVisualSpec): void {assert(Number.isSafeInteger(value.stackingOrder), "SpeechTrackVisualSpec stacking order is invalid");+ assertMediaFramePresentation(value.presentation, "SpeechTrackVisualSpec.presentation");+ assertMediaLifecycleMotion(value.motion, 1, "SpeechTrackVisualSpec.motion");+ sealMediaSampleLayerSpec({+ id: "speech-visual",+ appearance: value.sampleAppearance,+ ...(value.samplingMotion === undefined ? {} : { samplingMotion: value.samplingMotion }),+ });+ if (value.framePaint !== undefined) assertMediaPaintLayerSpec(value.framePaint);}export function assertSpeechTrackSet(value: SpeechTrackSet): void {@@ -107,7 +120,7 @@ export function appendSpeechTrackTake(...(semantic.media.visual === undefined ? {} : { visual: {frame: structuredClone(frame),fit: structuredClone(fit),- stackingOrder: visualSpec.stackingOrder,+ spec: structuredClone(visualSpec),} }),} satisfies SpeechTrackTake;return appendTake(set, take);diff --git a/packages/speech-track/src/projection.ts b/packages/speech-track/src/projection.tsindex 9d2ef9e1..e0b9cee9 100644--- a/packages/speech-track/src/projection.ts+++ b/packages/speech-track/src/projection.ts@@ -34,7 +34,7 @@ export function projectSpeechTrackVisual(track: SemanticTrack, set: SpeechTrackSframeCount: endFrameExclusive - startFrame,frame: placement.frame,fit: placement.fit,- stackingOrder: placement.stackingOrder,+ ...placement.spec,}];}),),diff --git a/packages/speech-track/src/surface.ts b/packages/speech-track/src/surface.tsindex a08af8a5..c1a05027 100644--- a/packages/speech-track/src/surface.ts+++ b/packages/speech-track/src/surface.ts@@ -8,6 +8,15 @@ import {} from "@hypit/markup";import { sameType } from "@hypit/protocol";import { speechTypes } from "@hypit/speech";+import {+ decodeMediaFramePaint,+ decodeMediaMotion,+ decodeMediaPresentation,+ decodeMediaSamplingKeyframe,+ decodeMediaSampleAppearance,+ mediaAppearanceKeys,+} from "@hypit/media-track";+import type { MediaSamplingMotion } from "@hypit/media-track";import type { SvsRecipe } from "@hypit/svs";import { svsRecipeType } from "@hypit/svs";@@ -57,15 +66,22 @@ function resolve(return value;}-function recipe(value: SurfaceResolvedReference, label: string): SvsRecipe {+const visualAppearancePropertyNames = [+ ...contentFitPropertyNames,+ ...mediaAppearanceKeys.sample.filter((name) => name !== "playback" && name !== "trim-start" && name !== "trim-end"),+ ...mediaAppearanceKeys.frame.filter((name) => name !== "stack-order"),+] as readonly string[];+const visualMotionPropertyNames = mediaAppearanceKeys.motion.filter(+ (name) => name !== "enter-origin" && name !== "exit-origin",+);++function recipe(value: SurfaceResolvedReference, label: string, allowed: readonly string[]): SvsRecipe {if (!sameType(value.type, svsRecipeType) || value.record?.value.kind !== "inline") {throw new Error(`${label} must be an authored SVS Recipe`);}const result = value.record.value.value as unknown as SvsRecipe;- const unknown = Object.keys(result.properties).filter((name) => !contentFitPropertyNames.includes(- name as (typeof contentFitPropertyNames)[number],- ));- if (unknown.length > 0) throw new Error(`${label} only accepts spatial fit properties; found ${unknown.join(", ")}`);+ const unknown = Object.keys(result.properties).filter((name) => !allowed.includes(name));+ if (unknown.length > 0) throw new Error(`${label} has unsupported properties ${unknown.join(", ")}`);return result;}@@ -83,16 +99,37 @@ function takes(element: StructuredElement): StructuredElement[] {return result;}+function samplingMotion(element: StructuredElement): MediaSamplingMotion | undefined {+ const keyframes = element.children.flatMap((child) => {+ if (child.kind === "text") {+ if (child.value.trim()) throw new Error(`${element.name} accepts only Sampling children`);+ return [];+ }+ if (localName(child.name) !== "Sampling") {+ throw new Error(`${element.name} accepts only Sampling children`);+ }+ return [decodeMediaSamplingKeyframe(child)];+ });+ return keyframes.length === 0 ? undefined : { keyframes };+}+export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {- allowedAttributes(element, ["id", "visual-frame", "visual-appearance", "visual-z"], []);+ allowedAttributes(element, ["id", "visual-frame", "visual-appearance", "visual-z"], ["visual-motion"]);const id = stringAttribute(element, "id");const baseFrame = resolve(element, "visual-frame", spatialTypes.frame, resolveReference);const baseAppearanceReference = resolve(element, "visual-appearance", svsRecipeType, resolveReference);const baseAppearance = recipe(baseAppearanceReference,`${element.name}.visual-appearance`,+ visualAppearancePropertyNames,);const baseZ = integerAttribute(element, "visual-z");+ const baseMotionReference = element.attributes["visual-motion"] === undefined+ ? undefined+ : resolve(element, "visual-motion", svsRecipeType, resolveReference);+ const baseMotion = baseMotionReference === undefined+ ? undefined+ : recipe(baseMotionReference, `${element.name}.visual-motion`, visualMotionPropertyNames);const frameInputNames = new Map<string, string>();const frameInputName = (value: SurfaceResolvedReference, fallback: string): string => {const key = JSON.stringify(value.ref);@@ -109,15 +146,16 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, refitInputNames.set(key, fallback);return fallback;};- const visualSpecInputNames = new Map<number, string>();- const visualSpecInputName = (stackingOrder: number, fallback: string): string => {- const previous = visualSpecInputNames.get(stackingOrder);+ const visualSpecInputNames = new Map<string, string>();+ const visualSpecInputName = (spec: ReturnType<typeof sealSpeechTrackVisualSpec>, fallback: string): string => {+ const key = JSON.stringify(spec);+ const previous = visualSpecInputNames.get(key);if (previous !== undefined) return previous;- visualSpecInputNames.set(stackingOrder, fallback);+ visualSpecInputNames.set(key, fallback);return fallback;};const declaredTakes = takes(element).map((take, index) => {- allowedAttributes(take, ["source"], ["frame", "appearance", "z"]);+ allowedAttributes(take, ["source"], ["frame", "appearance", "motion", "z"]);const suffix = String(index + 1).padStart(4, "0");const effectiveFrame = take.attributes.frame === undefined? baseFrame@@ -127,15 +165,31 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, re: resolve(take, "appearance", svsRecipeType, resolveReference);const effectiveAppearance = take.attributes.appearance === undefined? baseAppearance- : recipe(effectiveAppearanceReference, `${take.name}.appearance`);+ : recipe(effectiveAppearanceReference, `${take.name}.appearance`, visualAppearancePropertyNames);const effectiveZ = take.attributes.z === undefined ? baseZ : integerAttribute(take, "z");+ const effectiveMotionReference = take.attributes.motion === undefined+ ? baseMotionReference+ : resolve(take, "motion", svsRecipeType, resolveReference);+ const effectiveMotion = take.attributes.motion === undefined+ ? baseMotion+ : recipe(effectiveMotionReference!, `${take.name}.motion`, visualMotionPropertyNames);+ const effectiveSamplingMotion = samplingMotion(take);+ const framePaint = decodeMediaFramePaint(effectiveAppearance, `${id}.take.${suffix}.frame-paint`);+ const visualSpec = sealSpeechTrackVisualSpec({+ stackingOrder: effectiveZ,+ presentation: decodeMediaPresentation(effectiveAppearance),+ sampleAppearance: decodeMediaSampleAppearance(effectiveAppearance),+ motion: decodeMediaMotion(effectiveMotion),+ ...(framePaint === undefined ? {} : { framePaint }),+ ...(effectiveSamplingMotion === undefined ? {} : { samplingMotion: effectiveSamplingMotion }),+ });const visual = {frameName: frameInputName(effectiveFrame, take.attributes.frame === undefined ? "visual-base-frame" : `take-${suffix}-frame`),fitName: fitInputName(effectiveAppearanceReference, take.attributes.appearance === undefined ? "visual-base-fit" : `take-${suffix}-fit`),- visualSpecName: visualSpecInputName(effectiveZ, take.attributes.z === undefined ? "visual-base-spec" : `take-${suffix}-visual-spec`),+ visualSpecName: visualSpecInputName(visualSpec, `take-${suffix}-visual-spec`),frame: effectiveFrame,appearance: effectiveAppearance,- stackingOrder: effectiveZ,+ spec: visualSpec,};return {suffix,@@ -179,10 +233,7 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, rerecords.push({id: `${id}.${take.visual.visualSpecName}`,type: speechTrackTypes.visualSpec,- value: { kind: "inline" as const, value: sealSpeechTrackVisualSpec({-- stackingOrder: take.visual.stackingOrder,- }) },+ value: { kind: "inline" as const, value: take.visual.spec },range: take.range,});}diff --git a/packages/speech-track/src/types.ts b/packages/speech-track/src/types.tsindex 658b3ad1..567d684a 100644--- a/packages/speech-track/src/types.ts+++ b/packages/speech-track/src/types.ts@@ -1,4 +1,11 @@import type { ContentFit, SpatialFrame } from "@hypit/spatial";+import type {+ MediaFramePresentation,+ MediaLifecycleMotion,+ MediaPaintLayerSpec,+ MediaSampleAppearance,+ MediaSamplingMotion,+} from "@hypit/media-track";/** Author-owned policy for one ordered Speech Track. */export type SpeechTrackHeader = {@@ -10,12 +17,17 @@ export type SpeechTrackTake = {readonly visual?: {readonly frame: SpatialFrame;readonly fit: ContentFit;- readonly stackingOrder: number;+ readonly spec: SpeechTrackVisualSpec;};};export type SpeechTrackVisualSpec = {readonly stackingOrder: number;+ readonly presentation: MediaFramePresentation;+ readonly sampleAppearance: MediaSampleAppearance;+ readonly motion: MediaLifecycleMotion;+ readonly framePaint?: MediaPaintLayerSpec;+ readonly samplingMotion?: MediaSamplingMotion;};/**diff --git a/packages/speech-track/test/program.test.ts b/packages/speech-track/test/program.test.tsindex 218ff5b5..f4a26243 100644--- a/packages/speech-track/test/program.test.ts+++ b/packages/speech-track/test/program.test.ts@@ -52,7 +52,24 @@ function semantic(id: string, visual: boolean): SemanticTake {test("one Track assembles a continuous SemanticTrack and only visual Takes become presents", () => {const header = sealSpeechTrackHeader({ id: "speech" });- const visualSpec = sealSpeechTrackVisualSpec({ stackingOrder: 30 });+ const visualSpec = sealSpeechTrackVisualSpec({+ stackingOrder: 30,+ presentation: {+ clip: { kind: "rounded", radiusPx: 320 },+ padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 },+ border: { widthPx: 4, style: "solid", color: "#ffffff" },+ shadows: [],+ },+ sampleAppearance: {+ opacity: 1,+ filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 },+ },+ motion: { sustain: [] },+ samplingMotion: { keyframes: [+ { atProgress: 0, zoom: 1, offsetX: 0, offsetY: 0, rotationDeg: 0 },+ { atProgress: 1, zoom: 1.08, offsetX: 0, offsetY: -18, rotationDeg: 0 },+ ] },+ });let set = createSpeechTrackSet();set = appendSpeechTrackTake(set, semantic("voiceover", false), frame, fit, visualSpec);set = appendSpeechTrackTake(set, semantic("answer", true), frame, fit, visualSpec);@@ -64,4 +81,8 @@ test("one Track assembles a continuous SemanticTrack and only visual Takes becomassert.equal(visualTrack.presents.length, 1);assert.equal(visualTrack.presents[0]?.id, "speech:visual:answer");assert.deepEqual(visualTrack.presents[0]?.span, { startFrame: 30, endFrameExclusive: 60 });+ const styles = visualTrack.presents[0]?.elements.flatMap((element) => element.style) ?? [];+ assert.equal(styles.find((style) => style.name === "border-radius")?.value, "320px");+ assert.equal(styles.find((style) => style.name === "border")?.value, "4px solid #ffffff");+ assert.equal(visualTrack.presents[0]?.elements.some((element) => element.animation !== undefined), true);});diff --git a/packages/speech-track/test/surface.test.ts b/packages/speech-track/test/surface.test.tsindex 106d33a8..b7451f5f 100644--- a/packages/speech-track/test/surface.test.ts+++ b/packages/speech-track/test/surface.test.ts@@ -56,7 +56,10 @@ function resolved(path: string): SurfaceResolvedReference | undefined {ref: { kind: "record", id: path },type: svsRecipeType,record: { id: path, type: svsRecipeType, value: { kind: "inline", value: {- path: "speech.base", properties: { fit: "cover" },+ path: "speech.base", properties: {+ fit: "cover", clip: "rounded", radius: 360,+ "border-width": 4, "border-color": "#ffffff",+ },} } },};return undefined;@@ -76,7 +79,17 @@ test("Speech Track accepts only already-semantic Segment Takes", async () => {kind: "element",name: "speech:Take",attributes: { source: { kind: "reference", path: "opening.take" } },- children: [],+ children: [+ {+ kind: "element", name: "speech:Sampling",+ attributes: { at: "start", zoom: "1" }, children: [], range: { start: 30, end: 40 },+ },+ {+ kind: "element", name: "speech:Sampling",+ attributes: { at: "end", zoom: "1.08", y: "-18", easing: "ease-out" },+ children: [], range: { start: 41, end: 55 },+ },+ ],range: { start: 20, end: 60 },}],range: { start: 0, end: 70 },@@ -93,4 +106,17 @@ test("Speech Track accepts only already-semantic Segment Takes", async () => {audio: "speech.audio",});assert.equal(output.exports?.includes("speech.audio") ?? false, true);+ const visualSpec = output.records.find((record) => record.type.name === "SpeechTrackVisualSpec");+ assert.equal(visualSpec?.value.kind, "inline");+ if (visualSpec?.value.kind !== "inline") throw new Error("Speech Track visual spec was not materialized");+ const value = visualSpec.value.value as unknown as {+ presentation: { clip: { kind: string; radiusPx?: number }; border?: { widthPx: number } };+ samplingMotion?: { keyframes: readonly { atProgress: number; zoom: number; offsetY: number }[] };+ };+ assert.deepEqual(value.presentation.clip, { kind: "rounded", radiusPx: 360 });+ assert.equal(value.presentation.border?.widthPx, 4);+ assert.deepEqual(value.samplingMotion?.keyframes.map(({ atProgress, zoom, offsetY }) => ({ atProgress, zoom, offsetY })), [+ { atProgress: 0, zoom: 1, offsetY: 0 },+ { atProgress: 1, zoom: 1.08, offsetY: -18 },+ ]);});