Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat(speech): carry wordless segments through semantic takes

kashorin61bcc699c561
diff --git a/packages/media-track/README.md b/packages/media-track/README.md
index fc1a05a6..c8f19573 100644
--- a/packages/media-track/README.md
+++ b/packages/media-track/README.md
@@ -1,11 +1,10 @@
# `@hypit/media-track`
Place images, prepared video or compositable surfaces over the semantic timeline. B-roll is a common
-use: the picture can illustrate a phrase while the A-roll's speech continues. The same Track can
-present a speaking A-roll as an inset or a prepared transparent cutout; its SemanticTake continues
-to establish semantic time. The choice of visual component does not determine the performance's role.
-Items have independent Windows; a Sequence owns one visual slot whose Members replace each other
-through Handoffs.
+use: the picture can illustrate a phrase while the A-roll's speech continues. A prepared speaking
+A-roll is normally presented directly by Speech Track, including an inset or transparent cutout, so
+its semantic, visual and audio outputs keep one owner. Media Items own independently timed pictures;
+a Sequence owns one visual slot whose Members replace each other through Handoffs.
The Markup Track takes `semantic` and `canvas`. Its Surface projects authored Selections, Moments,
Segments or clock expressions into Windows and Instants. The component consumes those projected
diff --git a/packages/media-track/preview/build.svrun b/packages/media-track/preview/build.svrun
index 42e2994f..c1b5bdfc 100644
--- a/packages/media-track/preview/build.svrun
+++ b/packages/media-track/preview/build.svrun
@@ -3,18 +3,19 @@
<svrun version="1">
<author source="./preview.svml"/>
<import from="@hypit/stand-in@1" as="stand-in"/>
+ <import from="@hypit/semantic-take-estimate@1" as="estimate"/>
<target output="final.video"/>
- <fragment id="take-card" using="stand-in:video">
+ <fragment id="take-card" using="stand-in:timed-card">
<input name="canvas" from="vertical"/>
<input name="duration" value="8"/>
<input name="clock" from="clock"/>
</fragment>
- <fragment id="motion-a-card" using="stand-in:video">
+ <fragment id="motion-a-card" using="stand-in:timed-card">
<input name="canvas" from="vertical"/>
<input name="duration" value="5"/>
<input name="clock" from="clock"/>
</fragment>
- <fragment id="motion-b-card" using="stand-in:video">
+ <fragment id="motion-b-card" using="stand-in:timed-card">
<input name="canvas" from="vertical"/>
<input name="duration" value="5"/>
<input name="clock" from="clock"/>
@@ -22,4 +23,11 @@
<satisfy output="take.video" candidate="take-card.video"/>
<satisfy output="motion-a.video" candidate="motion-a-card.video"/>
<satisfy output="motion-b.video" candidate="motion-b-card.video"/>
+ <fragment id="take-timing" using="estimate:semantic-take">
+ <input name="narrative" from="story"/>
+ <input name="segment" from="story.segment.showcase"/>
+ <input name="media" from="take-preview-media.media"/>
+ <input name="policy" from="showcase-estimated.policy"/>
+ </fragment>
+ <satisfy output="showcase-semantic.take" candidate="take-timing.take"/>
</svrun>
diff --git a/packages/media-track/preview/preview.svml b/packages/media-track/preview/preview.svml
index 6926937b..c6ba09c8 100644
--- a/packages/media-track/preview/preview.svml
+++ b/packages/media-track/preview/preview.svml
@@ -15,7 +15,7 @@
in Studio as in a Build:
<import from="@hypit/stand-in@1" as="stand-in"/>
- <fragment id="a-card" using="stand-in:video">
+ <fragment id="a-card" using="stand-in:timed-card">
<input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/>
</fragment>
<satisfy output="motion-a.video" candidate="a-card.video"/>
@@ -29,6 +29,7 @@
<import as="pipeline" from="@hypit/media-pipeline@1"/>
<import as="speech" from="@hypit/speech-track@1"/>
<import as="whisperx" from="@hypit/whisperx@1"/>
+ <import as="estimate" from="@hypit/semantic-take-estimate@1"/>
<import as="media-track" from="@hypit/media-track@1"/>
<import as="space" from="@hypit/spatial@1"/>
<import as="program" from="@hypit/program-space@1"/>
@@ -59,12 +60,17 @@
<pipeline:Normalize id="take-media" source={take.video}
recipe={recipes.media.aroll} clock={clock}/>
+ <pipeline:Normalize id="take-preview-media" source={take.video} clock={clock}
+ video="primary-moving" audio="none" span-authority="video"/>
<pipeline:Normalize id="motion-a-media" source={motion-a.video}
recipe={recipes.media.broll} clock={clock}/>
<pipeline:Normalize id="motion-b-media" source={motion-b.video}
recipe={recipes.media.broll} clock={clock}/>
<whisperx:SemanticTake id="showcase-semantic" narrative={story}
segment={story.segment.showcase} media={take-media.media} language="en"/>
+ <estimate:SemanticTake id="showcase-estimated" narrative={story}
+ segment={story.segment.showcase} media={take-preview-media.media}
+ language="en" pace="normal" rounding="round"/>
<speech:Track id="speech"
visual-frame={speech-frame} visual-appearance={recipes.speech.visual} visual-z="0">
diff --git a/packages/media-track/src/author.ts b/packages/media-track/src/author.ts
index 5bacde1b..72b16404 100644
--- a/packages/media-track/src/author.ts
+++ b/packages/media-track/src/author.ts
@@ -16,6 +16,7 @@ import type {
MediaPaint,
MediaPaintLayerSpec,
MediaSampleLayerSpec,
+ MediaSampleAppearance,
MediaSequenceSpec,
MediaVisualOccupancy,
MediaVisualTrim,
@@ -175,17 +176,22 @@ export function decodeMediaSampleSpec(
id,
...(timed ? { occupancy: occupancy(recipe), ...(sourceTrim === undefined ? {} : { trim: sourceTrim }) } : {}),
- appearance: {
- opacity: number(recipe, "opacity", 1),
- filter: {
- blurPx: number(recipe, "blur", 0), brightness: number(recipe, "brightness", 1),
- contrast: number(recipe, "contrast", 1), saturation: number(recipe, "saturation", 1),
- },
- },
+ appearance: decodeMediaSampleAppearance(recipe),
...(samplingMotion === undefined ? {} : { samplingMotion }),
});
}
+/** Decode visual-only sample styling without choosing playback or source trim. */
+export function decodeMediaSampleAppearance(recipe: SvsRecipe): MediaSampleAppearance {
+ return {
+ opacity: number(recipe, "opacity", 1),
+ filter: {
+ blurPx: number(recipe, "blur", 0), brightness: number(recipe, "brightness", 1),
+ contrast: number(recipe, "contrast", 1), saturation: number(recipe, "saturation", 1),
+ },
+ };
+}
+
function edge(recipe: SvsRecipe, prefix: "enter" | "exit"): MediaLifecycleMotion["enter"] {
const operator = optionalString(recipe, prefix);
if (operator === undefined || operator === "none") return undefined;
diff --git a/packages/media-track/src/index.ts b/packages/media-track/src/index.ts
index 1ebb751d..9bc00617 100644
--- a/packages/media-track/src/index.ts
+++ b/packages/media-track/src/index.ts
@@ -4,6 +4,7 @@ export {
decodeMediaFramePaint,
decodeMediaMotion,
decodeMediaPresentation,
+ decodeMediaSampleAppearance,
decodeMediaSampleSpec,
mediaAppearanceKeys,
} from "./author.js";
@@ -23,7 +24,7 @@ export {
sealMediaSampleLayerSpec,
} from "./layers.js";
export { lowerMediaItemElements } from "./lower.js";
-export { mediaItemSpecSchema, mediaFramePresentationSchema, mediaLifecycleMotionSchema, mediaLayerSetSchema, mediaHandoffSpecSchema, mediaPaintLayerSpecSchema, mediaSampleLayerSpecSchema, mediaSoundSetSchema, mediaSoundSpecSchema, mediaSequenceMemberSetSchema, mediaSequenceMemberSpecSchema, mediaSequenceSpecSchema, mediaTrackDependency, mediaTrackHeaderSchema, mediaTrackManifest, mediaTrackMarkupSurfaces, mediaTrackModuleRef, mediaTrackProducers, mediaTrackProgramSchema, mediaTrackSetSchema, mediaTrackTypes } from "./manifest.js";
+export { mediaItemSpecSchema, mediaFramePresentationSchema, mediaLifecycleMotionSchema, mediaLayerSetSchema, mediaHandoffSpecSchema, mediaPaintLayerSpecSchema, mediaSampleAppearanceSchema, mediaSampleLayerSpecSchema, mediaSamplingMotionSchema, mediaSoundSetSchema, mediaSoundSpecSchema, mediaSequenceMemberSetSchema, mediaSequenceMemberSpecSchema, mediaSequenceSpecSchema, mediaTrackDependency, mediaTrackHeaderSchema, mediaTrackManifest, mediaTrackMarkupSurfaces, mediaTrackModuleRef, mediaTrackProducers, mediaTrackProgramSchema, mediaTrackSetSchema, mediaTrackTypes } from "./manifest.js";
export {
assertMediaEdgeMotion,
assertMediaLifecycleMotion,
@@ -47,7 +48,7 @@ export { appendMediaItem, appendMediaSequence, bindMediaItemClipPath, bindMediaS
export { resolveVisualSampling } from "./sampling.js";
export { lowerRestrictedSpeechVisualPresents } from "./restricted-speech.js";
export type { RestrictedSpeechVisualClip } from "./restricted-speech.js";
-export { decodeMediaTrackSurface } from "./surface.js";
+export { decodeMediaSamplingKeyframe, decodeMediaTrackSurface } from "./surface.js";
export {
appendMediaSequenceMember,
assertMediaHandoffSpec,
diff --git a/packages/media-track/src/manifest.ts b/packages/media-track/src/manifest.ts
index 76791c5e..48a8b8da 100644
--- a/packages/media-track/src/manifest.ts
+++ b/packages/media-track/src/manifest.ts
@@ -94,7 +94,7 @@ const paint: ValueSchema = { kind: "oneOf", variants: [
object({ kind: { schema: { kind: "literal", value: "linear-gradient" } }, angleDeg: { schema: number }, stops: { schema: stops } }),
object({ kind: { schema: { kind: "literal", value: "radial-gradient" } }, center: { schema: object({ x: { schema: unsigned }, y: { schema: unsigned } }) }, stops: { schema: stops } }),
] };
-const appearance = object({
+export const mediaSampleAppearanceSchema: ValueSchema = object({
opacity: { schema: unsigned },
filter: { schema: object({
blurPx: { schema: unsigned }, brightness: { schema: unsigned }, contrast: { schema: unsigned }, saturation: { schema: unsigned },
@@ -105,7 +105,9 @@ const samplingKeyframe = object({
offsetX: { schema: number }, offsetY: { schema: number }, rotationDeg: { schema: number },
easing: { schema: { kind: "string", enum: ["linear", "ease-in", "ease-out", "ease-in-out"] }, optional: true },
});
-const samplingMotion = object({ keyframes: { schema: { kind: "array", minItems: 2, items: samplingKeyframe } } });
+export const mediaSamplingMotionSchema: ValueSchema = object({
+ keyframes: { schema: { kind: "array", minItems: 2, items: samplingKeyframe } },
+});
export const mediaPaintLayerSpecSchema: ValueSchema = object({
id: { schema: string }, paint: { schema: paint }, opacity: { schema: unsigned },
@@ -113,7 +115,7 @@ export const mediaPaintLayerSpecSchema: ValueSchema = object({
export const mediaSampleLayerSpecSchema: ValueSchema = object({
id: { schema: string }, trim: { schema: trim, optional: true }, occupancy: { schema: occupancy, optional: true },
- appearance: { schema: appearance }, samplingMotion: { schema: samplingMotion, optional: true },
+ appearance: { schema: mediaSampleAppearanceSchema }, samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },
});
const audioSource = object({ artifact: { schema: blob }, sampleFrames: { schema: positiveInteger } });
const visualSource: ValueSchema = { kind: "oneOf", variants: [
@@ -128,7 +130,7 @@ const paintLayer = object({ id: { schema: string }, kind: { schema: { kind: "lit
const sampleLayer = object({
id: { schema: string }, kind: { schema: { kind: "literal", value: "sample" } }, source: { schema: visualSource },
fit: { schema: contentFitSchema }, trim: { schema: trim, optional: true }, occupancy: { schema: occupancy, optional: true },
- appearance: { schema: appearance }, samplingMotion: { schema: samplingMotion, optional: true },
+ appearance: { schema: mediaSampleAppearanceSchema }, samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },
});
const layer = { kind: "oneOf", variants: [paintLayer, sampleLayer] } as const;
export const mediaLayerSetSchema: ValueSchema = object({
diff --git a/packages/media-track/src/restricted-speech.ts b/packages/media-track/src/restricted-speech.ts
index 3d650fb9..f64cc7aa 100644
--- a/packages/media-track/src/restricted-speech.ts
+++ b/packages/media-track/src/restricted-speech.ts
@@ -6,7 +6,14 @@ import { assertContentFit, assertIntrinsicExtent, assertSpatialFrame } from "@hy
import type { ContentFit, IntrinsicExtent, SpatialFrame } from "@hypit/spatial";
import { lowerMediaItemElements } from "./lower.js";
-import type { MediaItemProgram } from "./types.js";
+import type {
+ MediaFramePresentation,
+ MediaItemProgram,
+ MediaLifecycleMotion,
+ MediaPaintLayerSpec,
+ MediaSampleAppearance,
+ MediaSamplingMotion,
+} from "./types.js";
export type RestrictedSpeechVisualClip = {
readonly id: string;
@@ -19,12 +26,17 @@ export type RestrictedSpeechVisualClip = {
readonly frame: SpatialFrame;
readonly fit: ContentFit;
readonly stackingOrder: number;
+ readonly presentation: MediaFramePresentation;
+ readonly sampleAppearance: MediaSampleAppearance;
+ readonly motion: MediaLifecycleMotion;
+ readonly framePaint?: MediaPaintLayerSpec;
+ readonly samplingMotion?: MediaSamplingMotion;
};
/**
- * Focused reuse point for Speech Track: one normalized muted take and one
- * explicitly placed foreground layer, without Media Track motion or sequence
- * semantics.
+ * Focused reuse point for Speech Track: one normalized muted Take and its
+ * visual presentation inside a fixed Segment envelope, without Media Track
+ * Window, playback/trim, source-audio or Sequence ownership.
*/
export function lowerRestrictedSpeechVisualPresents(
trackId: string,
@@ -57,29 +69,33 @@ export function lowerRestrictedSpeechVisualPresents(
subjectId: clip.subjectId ?? clip.id,
span: { ...clip.span },
frame: structuredClone(clip.frame),
- presentation: {
- clip: { kind: "frame" },
- padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 },
- shadows: [],
- },
- layers: [{
- id: `${trackId}:${clip.id}:foreground`,
- kind: "sample",
- source: {
- kind: "timed",
- artifact: structuredClone(clip.artifact),
- extent: structuredClone(clip.extent),
- frameRate: { ...clip.frameRate },
- frameCount: clip.frameCount,
- },
- fit: structuredClone(clip.fit),
- occupancy: { mode: "once", align: "start" },
- appearance: {
- opacity: 1,
- filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 },
+ presentation: structuredClone(clip.presentation),
+ layers: [
+ ...(clip.framePaint === undefined ? [] : [{
+ id: clip.framePaint.id,
+ kind: "paint" as const,
+ paint: structuredClone(clip.framePaint.paint),
+ opacity: clip.framePaint.opacity,
+ }]),
+ {
+ id: `${trackId}:${clip.id}:foreground`,
+ kind: "sample",
+ source: {
+ kind: "timed",
+ artifact: structuredClone(clip.artifact),
+ extent: structuredClone(clip.extent),
+ frameRate: { ...clip.frameRate },
+ frameCount: clip.frameCount,
+ },
+ fit: structuredClone(clip.fit),
+ occupancy: { mode: "once", align: "start" },
+ appearance: structuredClone(clip.sampleAppearance),
+ ...(clip.samplingMotion === undefined ? {} : {
+ samplingMotion: structuredClone(clip.samplingMotion),
+ }),
},
- }],
- motion: { sustain: [] },
+ ],
+ motion: structuredClone(clip.motion),
stacking: {
order: clip.stackingOrder,
tieBreak: `${trackId}:${String(index + 1).padStart(4, "0")}:${clip.id}`,
diff --git a/packages/media-track/src/surface.ts b/packages/media-track/src/surface.ts
index 6b9a33a7..c8053241 100644
--- a/packages/media-track/src/surface.ts
+++ b/packages/media-track/src/surface.ts
@@ -307,7 +307,7 @@ function declaredVisualSource(
: { kind, value: reference(element.attributes.surface, `${element.name}.surface`, mediaTypes.compositableSurface, resolve) };
}
-function sampling(element: StructuredElement): MediaSamplingMotion["keyframes"][number] {
+export function decodeMediaSamplingKeyframe(element: StructuredElement): MediaSamplingMotion["keyframes"][number] {
allowed(element, ["at", "zoom", "x", "y", "rotate", "easing"]);
empty(element);
const at = text(element, "at");
@@ -348,7 +348,7 @@ function sourceLayer(
if (context.ignoreSiblingChildren === true) return [];
throw new Error(`${element.name} accepts only Sampling children.`);
}
- return [sampling(child)];
+ return [decodeMediaSamplingKeyframe(child)];
});
const motion = keyframes.length === 0 ? undefined : { keyframes };
const layerSuffix = `${context.unitSuffix}-layer-${suffix(layerIndex)}`;
diff --git a/packages/narrative/src/identity.ts b/packages/narrative/src/identity.ts
index 583ef29a..1af8abc2 100644
--- a/packages/narrative/src/identity.ts
+++ b/packages/narrative/src/identity.ts
@@ -38,7 +38,7 @@ export function assertNarrativeIdentity(value: Narrative): void {
let tokenCursor = 0;
for (const segment of value.segments) {
if (!Number.isSafeInteger(segment.tokenStart) || !Number.isSafeInteger(segment.tokenEndExclusive)
- || segment.tokenStart !== tokenCursor || segment.tokenEndExclusive <= segment.tokenStart
+ || segment.tokenStart !== tokenCursor || segment.tokenEndExclusive < segment.tokenStart
|| segment.tokenEndExclusive > value.tokens.length
|| !anchorIds.has(segment.startAnchorId) || !anchorIds.has(segment.endAnchorId)) {
throw new Error(`Narrative Segment ${segment.id} has invalid token or Anchor boundaries.`);
@@ -79,7 +79,7 @@ export function assertNarrativeExcerptIdentity(value: NarrativeExcerpt): void {
nonempty(value.narrativeId, "NarrativeExcerpt narrativeId");
nonempty(value.id, "NarrativeExcerpt id");
if (!Number.isSafeInteger(value.tokenStart) || !Number.isSafeInteger(value.tokenEndExclusive)
- || value.tokenStart < 0 || value.tokenEndExclusive <= value.tokenStart) {
+ || value.tokenStart < 0 || value.tokenEndExclusive < value.tokenStart) {
throw new Error("NarrativeExcerpt Token coverage is invalid.");
}
}
@@ -100,7 +100,9 @@ export function assertNarrativeMomentRefIdentity(value: NarrativeMomentRef): voi
export function assertCaptionDocumentIdentity(value: CaptionDocument): void {
nonempty(value.narrativeId, "CaptionDocument narrativeId");
nonempty(value.id, "CaptionDocument id");
- if (value.units.length === 0 || value.words.length === 0) throw new Error("CaptionDocument is empty.");
+ if ((value.units.length === 0) !== (value.words.length === 0)) {
+ throw new Error("CaptionDocument units and words must be empty together.");
+ }
const unitIds = unique(value.units.map((item) => item.id), "CaptionDocument unit id");
const wordIds = unique(value.words.map((item) => item.id), "CaptionDocument word id");
const words = new Map(value.words.map((word) => [word.id, word] as const));
diff --git a/packages/narrative/src/schema.ts b/packages/narrative/src/schema.ts
index b86f8b14..534b545b 100644
--- a/packages/narrative/src/schema.ts
+++ b/packages/narrative/src/schema.ts
@@ -73,8 +73,8 @@ const captionCueBreak = object({ afterUnitId: { schema: string } });
export const captionDocumentSchema: ValueSchema = object({
narrativeId: { schema: string },
id: { schema: string },
- units: { schema: { kind: "array", minItems: 1, items: captionAlignmentUnit } },
- words: { schema: { kind: "array", minItems: 1, items: captionDisplayWord } },
+ units: { schema: { kind: "array", items: captionAlignmentUnit } },
+ words: { schema: { kind: "array", items: captionDisplayWord } },
cueBreaks: { schema: { kind: "array", items: captionCueBreak } },
});
export const narrativeMomentSchema: ValueSchema = object({
diff --git a/packages/provider-hyperframes-local/test/alpha.test.ts b/packages/provider-hyperframes-local/test/alpha.test.ts
index 765c28b0..5ddcc98a 100644
--- a/packages/provider-hyperframes-local/test/alpha.test.ts
+++ b/packages/provider-hyperframes-local/test/alpha.test.ts
@@ -44,7 +44,12 @@ test("transparent normalized media composites through both SemanticTake/Speech T
const fit = { sizing: "contain" as const, framePoint: { x: 0.5, y: 0.5 }, contentPoint: { x: 0.5, y: 0.5 },
offsetPx: { x: 0, y: 0 }, constraint: "bounded" as const };
const frame = { xPx: 0, yPx: 0, widthPx: 96, heightPx: 64 };
- const set = appendSpeechTrackTake(createSpeechTrackSet(), take, frame, fit, sealSpeechTrackVisualSpec({ stackingOrder: 1 }));
+ const set = appendSpeechTrackTake(createSpeechTrackSet(), take, frame, fit, sealSpeechTrackVisualSpec({
+ stackingOrder: 1,
+ presentation: { clip: { kind: "frame" }, padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 }, shadows: [] },
+ sampleAppearance: { opacity: 1, filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 } },
+ motion: { sustain: [] },
+ }));
const semantic = assembleSpeechTrack(sealSpeechTrackHeader({ id: "speech" }), set);
const speechVisual = projectSpeechTrackVisual(semantic, set);
const space = projectSemanticProgramSpace(semantic);
diff --git a/packages/semantic-take-estimate/README.md b/packages/semantic-take-estimate/README.md
index faad4958..354ddc41 100644
--- a/packages/semantic-take-estimate/README.md
+++ b/packages/semantic-take-estimate/README.md
@@ -71,5 +71,5 @@ the `opening-estimated` declaration above, a Run can use:
This selects estimated timing for the measured Take while retaining the required prepared-media
dependency and any Run choice for its source bytes. The estimate allocates the media's complete
-frame count; it neither generates nor normalizes that media. It requires spoken Tokens. A wordless
-Segment uses actual media boundaries directly when materializing its SemanticTake.
+frame count; it neither generates nor normalizes that media. For a wordless Segment it maps the
+Segment's two boundary Anchors directly to the media's first and final frame.
diff --git a/packages/semantic-take-estimate/src/manifest.ts b/packages/semantic-take-estimate/src/manifest.ts
index d73642f5..cfce1421 100644
--- a/packages/semantic-take-estimate/src/manifest.ts
+++ b/packages/semantic-take-estimate/src/manifest.ts
@@ -18,7 +18,7 @@ export const semanticTakeEstimateMarkupSurfaces = [{
outputs: [estimateTypes.speechPolicy, speechTypes.semanticTake],
vocabulary: {
summary:
- "Projects syllable-weighted estimated word windows onto one already normalized video and publishes a SemanticTake.",
+ "Projects estimated word windows or wordless Segment boundaries onto normalized media and publishes a SemanticTake.",
attributes: [
{ name: "id", kind: "identifier", required: true,
summary: "Names this estimated alignment and the SemanticTake it publishes." },
@@ -57,7 +57,7 @@ export const semanticTakeEstimateMarkupSurfaces = [{
notes: [
"The policy is the element's own: write it inline or name an SVS Recipe with policy; never both.",
"This package does not measure audio and never claims WhisperX or other acoustic evidence.",
- "Every Token uses its original Script identity. Only its local frame window is estimated.",
+ "Every spoken Token uses its original Script identity. A wordless Segment keeps its two authored boundary Anchors.",
"Pronunciation-unit weights determine relative word lengths; adjacent words and both media edges retain non-zero frame gaps.",
"The result has the same SemanticTake Type as a measured alignment, so Run chooses between ordinary candidates without a preview-specific downstream protocol.",
],
diff --git a/packages/semantic-take-estimate/src/program.ts b/packages/semantic-take-estimate/src/program.ts
index 3afc64e7..36e3d0ad 100644
--- a/packages/semantic-take-estimate/src/program.ts
+++ b/packages/semantic-take-estimate/src/program.ts
@@ -22,7 +22,6 @@ function segmentTokens(narrative: Narrative, excerpt: NarrativeExcerpt): readonl
throw new Error(`NarrativeExcerpt ${excerpt.id} does not describe its authored Segment.`);
}
const tokens = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);
- if (tokens.length === 0) throw new Error(`Estimated SemanticTake Segment ${excerpt.id} contains no speech Tokens.`);
return tokens;
}
@@ -49,6 +48,21 @@ export function estimateSemanticTakeTiming(
verifySynchronizedMedia(media);
assertSpeechEstimatePolicy(policy);
const tokens = segmentTokens(narrative, excerpt);
+ const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id)!;
+ if (tokens.length === 0) {
+ const frameByAnchor = new Map<string, number>([
+ [segment.startAnchorId, 0],
+ [segment.endAnchorId, media.timeline.frameCount],
+ ]);
+ const anchors = narrative.semanticIndex.anchors
+ .filter((anchor) => anchor.segmentId === excerpt.id)
+ .map((anchor) => {
+ const frame = frameByAnchor.get(anchor.id);
+ if (frame === undefined) throw new Error(`Estimated timing cannot locate authored Anchor ${anchor.id}.`);
+ return { identity: anchor.id, frame };
+ });
+ return { tokens: [], anchors };
+ }
const rate = media.timeline.frameRate.numerator / media.timeline.frameRate.denominator;
const edgeGap = Math.max(1, Math.round(EDGE_GAP_SECONDS * rate));
const wordGap = Math.max(1, Math.round(WORD_GAP_SECONDS * rate));
@@ -79,7 +93,6 @@ export function estimateSemanticTakeTiming(
throw new Error("Estimated SemanticTake timing allocation did not close over its frame domain.");
}
- const segment = narrative.segments.find((candidate) => candidate.id === excerpt.id)!;
const frameByAnchor = new Map<string, number>([
[segment.startAnchorId, 0],
[segment.endAnchorId, media.timeline.frameCount],
diff --git a/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts b/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts
index 431385b3..d72a9f70 100644
--- a/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts
+++ b/packages/semantic-take-estimate/test/semantic-take-estimate.test.ts
@@ -3,6 +3,11 @@ import test from "node:test";
import { sealSpeechEstimatePolicy } from "@hypit/estimate";
import { sealSynchronizedMedia } from "@hypit/media";
+import {
+ assertCaptionDocumentIdentity,
+ assertNarrativeExcerptIdentity,
+ assertNarrativeIdentity,
+} from "@hypit/narrative";
import type { Narrative } from "@hypit/narrative";
import { fixtureResource } from "../../../test/fixture-resource.js";
@@ -106,3 +111,50 @@ test("an implausibly short frame domain is refused instead of squeezing words to
/keep every Token visible and separated/u,
);
});
+
+test("a wordless Segment receives the prepared media boundaries without an audio substitute", () => {
+ const wordless: Narrative = {
+ id: "wordless",
+ segments: [{
+ id: "pause",
+ startAnchorId: "pause:start",
+ endAnchorId: "pause:end",
+ tokenStart: 0,
+ tokenEndExclusive: 0,
+ }],
+ tokens: [],
+ turns: [],
+ selections: [],
+ moments: [],
+ semanticIndex: { anchors: [
+ { id: "program:start", kind: "program-start" },
+ { id: "pause:start", kind: "segment-start", segmentId: "pause" },
+ { id: "pause:end", kind: "segment-end", segmentId: "pause" },
+ { id: "program:end", kind: "program-end" },
+ ] },
+ };
+ const wordlessExcerpt = {
+ narrativeId: wordless.id,
+ kind: "segment" as const,
+ id: "pause",
+ tokenStart: 0,
+ tokenEndExclusive: 0,
+ };
+ assertNarrativeIdentity(wordless);
+ assertNarrativeExcerptIdentity(wordlessExcerpt);
+ assertCaptionDocumentIdentity({
+ narrativeId: wordless.id,
+ id: "wordless.caption",
+ units: [],
+ words: [],
+ cueBreaks: [],
+ });
+
+ const take = materializeEstimatedSemanticTake(wordless, wordlessExcerpt, media, policy);
+ assert.deepEqual(take.tokens, []);
+ assert.deepEqual(take.anchors, [
+ { identity: "pause:start", frame: 0 },
+ { identity: "pause:end", frame: media.timeline.frameCount },
+ ]);
+ assert.equal(take.media.audio, undefined);
+});
diff --git a/packages/speech-track/README.md b/packages/speech-track/README.md
index c88ed32f..e9075c28 100644
--- a/packages/speech-track/README.md
+++ b/packages/speech-track/README.md
@@ -7,8 +7,9 @@ speaking turns share this timeline; an independent narration can also carry it.
A-roll can be a circular picture-in-picture or a foreground presenter cutout over another main
picture. Its speaking performance establishes the semantic timeline regardless of its screen area
-or stack order. Speech Track supplies that timeline and sound even when Media Track or a project
-component supplies the chosen presentation of the same person.
+or stack order. Speech Track projects that same prepared Take into semantic, visual and audio
+outputs together; the inset or cutout does not need to be displayed a second time through Media
+Track.
## Assemble prepared Takes
@@ -26,9 +27,22 @@ contain several speakers or camera cuts. Takes are assembled in Source order:
</speech:Track>
```
-This excerpt assumes the Takes, Frame and appearance Recipe exist. A fit Recipe can be
-`performance { fit: cover; }`. The shared frame rate and total ProgramSpace come from the ordered
-Takes. Their local word and boundary frames become positions in that assembled space.
+This excerpt assumes the Takes, Frame and appearance Recipe exist. An appearance Recipe can share
+the ordinary visual appearance of a Media Item:
+
+```svs
+performance {
+ fit: cover;
+ clip: rounded;
+ radius: 270;
+ border-width: 3;
+ border-color: rgba(255, 255, 255, 0.9);
+}
+```
+
+With a `540 × 540` Frame, that rounded clip presents the performance as a circle. The shared frame
+rate and total ProgramSpace come from the ordered Takes. Their local word and boundary frames become
+positions in that assembled space.
| Output | Use |
| --- | --- |
@@ -36,16 +50,36 @@ Takes. Their local word and boundary frames become positions in that assembled s
| `.visual` | Include the performance picture in Film when wanted |
| `.audio` | Include the performance sound in Film, including beneath B-roll |
+Each Take always contributes its Segment and Anchor timing. A Take with no audio stream contributes
+no audio clip; a Take with no visual stream contributes no picture. Its remaining semantic and
+media projections continue normally, so a video-only wordless Take needs no synthetic audio and an
+audio-only narration needs no synthetic picture.
+
Select each wanted picture and sound output explicitly in Film. Replacing the visible picture with
Media Track coverage does not replace the semantic timeline or mute the included performance audio.
An audio-only performance can supply the timeline while other Tracks supply the picture.
-## Place the performance and extend its presentation
+## Present the performance directly
+
+`visual-frame`, `visual-appearance`, `visual-motion` and `visual-z` supply the Track's visual base. A
+Take can override them with `frame`, `appearance`, `motion` and `z`. Appearance can control fit and
+crop anchors, opacity and filters, rounded or rectangular clipping, padding, borders, shadows and
+frame paint. Lifecycle motion can move that visual presentation as it enters, remains or exits. None
+of these choices changes the Take's semantic timing or audio.
+
+Sampling children move the Take's picture inside its Frame over normalized Segment progress, using
+the same fields as a direct Media Item:
+
+```svml
+<speech:Take source={opening.take}>
+ <speech:Sampling at="start" zoom="1"/>
+ <speech:Sampling at="end" zoom="1.08" y="-18" easing="ease-out"/>
+</speech:Take>
+```
-`visual-frame`, `visual-appearance` and `visual-z` supply the Track's visual base. A Take can override
-it with `frame`, `appearance` and `z`. This places and stacks the same performance picture. Use
-[Media Track](../media-track/README.md) for an independently animated presentation, replacement
-Sequence or transition. Caption and MG consume `.semantic` to follow the same words and events.
+Use [Media Track](../media-track/README.md) when a picture owns an independent semantic Window,
+source playback or trim, source audio, a replacement Sequence, or a visual source other than the
+performance itself. Caption and MG consume `.semantic` to follow the same words and events.
The [Studio Companion](../speech-track-studio/src/index.ts) presents the assembled Takes and their
materials. Media preparation is owned by [Media Pipeline](../media-pipeline/README.md), and exact
diff --git a/packages/speech-track/src/manifest.ts b/packages/speech-track/src/manifest.ts
index bc36425b..77e04c77 100644
--- a/packages/speech-track/src/manifest.ts
+++ b/packages/speech-track/src/manifest.ts
@@ -2,6 +2,13 @@ import { readFile } from "node:fs/promises";
import { semanticTakeSchema, speechDependency, speechTypes } from "@hypit/speech";
import { compositionDependency, compositionTypes } from "@hypit/composition";
+import {
+ mediaFramePresentationSchema,
+ mediaLifecycleMotionSchema,
+ mediaPaintLayerSpecSchema,
+ mediaSampleAppearanceSchema,
+ mediaSamplingMotionSchema,
+} from "@hypit/media-track";
import type { ModuleManifest, ProducerRef, TypeRef, ValueSchema } from "@hypit/protocol";
import { semanticTrackDependency, semanticTrackTypes } from "@hypit/semantic-track";
import {
@@ -41,21 +48,26 @@ export const speechTrackHeaderSchema: ValueSchema = object({
id: { schema: string },
});
+export const speechTrackVisualSpecSchema: ValueSchema = object({
+ stackingOrder: { schema: { kind: "number", integer: true } },
+ presentation: { schema: mediaFramePresentationSchema },
+ sampleAppearance: { schema: mediaSampleAppearanceSchema },
+ motion: { schema: mediaLifecycleMotionSchema },
+ framePaint: { schema: mediaPaintLayerSpecSchema, optional: true },
+ samplingMotion: { schema: mediaSamplingMotionSchema, optional: true },
+});
+
export const speechTrackSetSchema: ValueSchema = object({
takes: { schema: { kind: "array", items: object({
semantic: { schema: semanticTakeSchema },
visual: { schema: object({
frame: { schema: spatialFrameSchema },
fit: { schema: contentFitSchema },
- stackingOrder: { schema: { kind: "number", integer: true } },
+ spec: { schema: speechTrackVisualSpecSchema },
}), optional: true },
}) } },
});
-export const speechTrackVisualSpecSchema: ValueSchema = object({
- stackingOrder: { schema: { kind: "number", integer: true } },
-});
-
const visualRecipeProperties = [
{ name: "fit", required: false, fallback: "contain",
values: ["contain", "cover", "fit-width", "fit-height", "native", "scale-down", "stretch"],
@@ -74,6 +86,59 @@ const visualRecipeProperties = [
summary: "Shifts the placed picture vertically in pixels once the two anchor points meet." },
{ name: "fit-constraint", required: false, fallback: "bounded", values: ["bounded", "free"],
summary: "Decides whether the placed picture is pulled back until it covers as much of the Frame as its size allows, or left exactly where the anchors and offsets put it." },
+ { name: "opacity", required: false, fallback: "1",
+ summary: "Sets how opaque the Take's picture is drawn." },
+ { name: "blur", required: false, fallback: "0",
+ summary: "Blurs the Take's picture by a pixel radius." },
+ { name: "brightness", required: false, fallback: "1",
+ summary: "Scales the brightness of the Take's picture." },
+ { name: "contrast", required: false, fallback: "1",
+ summary: "Scales the contrast of the Take's picture." },
+ { name: "saturation", required: false, fallback: "1",
+ summary: "Scales the saturation of the Take's picture." },
+ { name: "clip", required: false, values: ["none", "frame", "rounded"], fallback: "frame",
+ summary: "Clips the Take's picture to its Frame, leaves overflow visible, or rounds the Frame." },
+ { name: "radius", required: false, fallback: "0",
+ summary: "Sets the rounded clip radius in pixels; half the side of a square Frame makes a circle." },
+ { name: "padding", required: false, fallback: "0",
+ summary: "Insets the Take's picture from its Frame edges using one, two or four pixel values." },
+ { name: "border-width", required: false, fallback: "0",
+ summary: "Draws a border of this pixel width around the Frame." },
+ { name: "border-style", required: false, values: ["solid", "dashed", "dotted"], fallback: "solid",
+ summary: "Chooses how a non-zero border is stroked." },
+ { name: "border-color", required: false,
+ summary: "Sets the color of a non-zero border." },
+ { name: "shadows", required: false, fallback: "none",
+ summary: "Casts Frame shadows written as `x y blur spread color` entries separated by semicolons." },
+ { name: "frame-paint", required: false, fallback: "transparent",
+ summary: "Fills the Frame behind the Take with a solid or gradient paint." },
+] as const;
+
+const visualMotionRecipeProperties = [
+ { name: "enter", required: false, fallback: "none",
+ values: ["none", "fade", "slide", "scale", "pop", "bounce", "blur-reveal", "wipe", "flip", "spin"],
+ summary: "Chooses the visual entrance applied inside this Segment." },
+ { name: "enter-frames", required: false,
+ summary: "Sets the entrance length in frames when an entrance is selected." },
+ { name: "enter-easing", required: false, values: ["linear", "ease-in", "ease-out", "ease-in-out"], fallback: "ease-in-out",
+ summary: "Shapes the entrance's acceleration." },
+ { name: "enter-direction", required: false, values: ["left", "right", "up", "down"],
+ summary: "Sets the direction of a directional entrance." },
+ { name: "enter-amount", required: false,
+ summary: "Sets how far the entrance displaces the visual." },
+ { name: "sustain", required: false, fallback: "none",
+ summary: "Applies continuous visual motion over the Segment as `operator amount cycles [direction]`." },
+ { name: "exit", required: false, fallback: "none",
+ values: ["none", "fade", "slide", "scale", "pop", "bounce", "blur-reveal", "wipe", "flip", "spin"],
+ summary: "Chooses the visual exit applied inside this Segment." },
+ { name: "exit-frames", required: false,
+ summary: "Sets the exit length in frames when an exit is selected." },
+ { name: "exit-easing", required: false, values: ["linear", "ease-in", "ease-out", "ease-in-out"], fallback: "ease-in-out",
+ summary: "Shapes the exit's acceleration." },
+ { name: "exit-direction", required: false, values: ["left", "right", "up", "down"],
+ summary: "Sets the direction of a directional exit." },
+ { name: "exit-amount", required: false,
+ summary: "Sets how far the exit displaces the visual." },
] as const;
export const speechTrackMarkupSurfaces = [{
@@ -97,10 +162,13 @@ export const speechTrackMarkupSurfaces = [{
recipe: visualRecipeProperties },
{ name: "visual-z", kind: "literal", required: true,
summary: "Sets the stacking order every visual Take is composited at unless the Take names its own." },
+ { name: "visual-motion", kind: "reference", required: false, accepts: [svsRecipeType],
+ summary: "Chooses visual-only entrance, sustained motion and exit for every Take unless a Take names its own.",
+ recipe: visualMotionRecipeProperties },
],
children: [
{ tag: "Take", cardinality: "many",
- summary: "One self-contained SemanticTake, in document order.",
+ summary: "One self-contained SemanticTake, in document order; optional Sampling children move its picture inside the Frame.",
attributes: [
{ name: "source", kind: "reference", required: true, accepts: [speechTypes.semanticTake],
summary: "Chooses the normalized and locally aligned SemanticTake this Track assembles." },
@@ -111,6 +179,9 @@ export const speechTrackMarkupSurfaces = [{
recipe: visualRecipeProperties },
{ name: "z", kind: "literal", required: false,
summary: "Sets this Take's own stacking order in place of the Track's `visual-z`." },
+ { name: "motion", kind: "reference", required: false, accepts: [svsRecipeType],
+ summary: "Chooses this Take's own visual-only motion in place of the Track's `visual-motion`.",
+ recipe: visualMotionRecipeProperties },
] },
],
ports: [
@@ -128,9 +199,11 @@ export const speechTrackMarkupSurfaces = [{
</speech:Track>`,
notes: [
"A Track requires at least one Take and accepts no text content.",
- "`visual-appearance` and a Take's `appearance` must each be an authored SVS Recipe declaring only the spatial fit properties listed for them; any other property is refused.",
+ "`visual-appearance` and a Take's `appearance` use the shared Media Item styling listed here: fit, visual filtering, Frame paint, clipping, border and shadow.",
"Media normalization, acoustic evidence and Segment alignment happen before a Take enters the Track.",
- "Placement and stacking are the complete visual authority of a Track; motion, transitions and independent pictures remain ordinary Media Tracks.",
+ "Speech visual motion changes only the projected picture inside its Segment; it does not retime the SemanticTake or its audio.",
+ "A Take accepts Sampling children with the same `at`, `zoom`, `x`, `y`, `rotate` and `easing` fields as a direct Media Item; at least two keyframes cover normalized Segment progress.",
+ "Playback, source trim, independent Windows, replacement Sequences and separate pictures remain ordinary Media Track concerns.",
],
},
}] as const;
diff --git a/packages/speech-track/src/program.ts b/packages/speech-track/src/program.ts
index 1f624e91..0c8a0e4d 100644
--- a/packages/speech-track/src/program.ts
+++ b/packages/speech-track/src/program.ts
@@ -3,6 +3,12 @@ import type { SemanticTake } from "@hypit/speech";
import { sealSemanticTrack } from "@hypit/semantic-track";
import type { SemanticTrack } from "@hypit/semantic-track";
import { canonicalize } from "@hypit/protocol";
+import {
+ assertMediaFramePresentation,
+ assertMediaLifecycleMotion,
+ assertMediaPaintLayerSpec,
+ sealMediaSampleLayerSpec,
+} from "@hypit/media-track";
import { assertContentFit, assertSpatialFrame } from "@hypit/spatial";
import type { ContentFit, SpatialFrame } from "@hypit/spatial";
@@ -47,8 +53,7 @@ function assertTake(take: SpeechTrackTake): void {
if (take.visual !== undefined) {
assertSpatialFrame(take.visual.frame);
assertContentFit(take.visual.fit);
- assert(Number.isSafeInteger(take.visual.stackingOrder),
- `Speech Segment ${segment.segmentId} visual stacking order is invalid`);
+ assertSpeechTrackVisualSpec(take.visual.spec);
}
}
@@ -60,6 +65,14 @@ export function sealSpeechTrackVisualSpec(value: SpeechTrackVisualSpec): SpeechT
export function assertSpeechTrackVisualSpec(value: SpeechTrackVisualSpec): void {
assert(Number.isSafeInteger(value.stackingOrder), "SpeechTrackVisualSpec stacking order is invalid");
+ assertMediaFramePresentation(value.presentation, "SpeechTrackVisualSpec.presentation");
+ assertMediaLifecycleMotion(value.motion, 1, "SpeechTrackVisualSpec.motion");
+ sealMediaSampleLayerSpec({
+ id: "speech-visual",
+ appearance: value.sampleAppearance,
+ ...(value.samplingMotion === undefined ? {} : { samplingMotion: value.samplingMotion }),
+ });
+ if (value.framePaint !== undefined) assertMediaPaintLayerSpec(value.framePaint);
}
export function assertSpeechTrackSet(value: SpeechTrackSet): void {
@@ -107,7 +120,7 @@ export function appendSpeechTrackTake(
...(semantic.media.visual === undefined ? {} : { visual: {
frame: structuredClone(frame),
fit: structuredClone(fit),
- stackingOrder: visualSpec.stackingOrder,
+ spec: structuredClone(visualSpec),
} }),
} satisfies SpeechTrackTake;
return appendTake(set, take);
diff --git a/packages/speech-track/src/projection.ts b/packages/speech-track/src/projection.ts
index 9d2ef9e1..e0b9cee9 100644
--- a/packages/speech-track/src/projection.ts
+++ b/packages/speech-track/src/projection.ts
@@ -34,7 +34,7 @@ export function projectSpeechTrackVisual(track: SemanticTrack, set: SpeechTrackS
frameCount: endFrameExclusive - startFrame,
frame: placement.frame,
fit: placement.fit,
- stackingOrder: placement.stackingOrder,
+ ...placement.spec,
}];
}),
),
diff --git a/packages/speech-track/src/surface.ts b/packages/speech-track/src/surface.ts
index a08af8a5..c1a05027 100644
--- a/packages/speech-track/src/surface.ts
+++ b/packages/speech-track/src/surface.ts
@@ -8,6 +8,15 @@ import {
} from "@hypit/markup";
import { sameType } from "@hypit/protocol";
import { speechTypes } from "@hypit/speech";
+import {
+ decodeMediaFramePaint,
+ decodeMediaMotion,
+ decodeMediaPresentation,
+ decodeMediaSamplingKeyframe,
+ decodeMediaSampleAppearance,
+ mediaAppearanceKeys,
+} from "@hypit/media-track";
+import type { MediaSamplingMotion } from "@hypit/media-track";
import type { SvsRecipe } from "@hypit/svs";
import { svsRecipeType } from "@hypit/svs";
@@ -57,15 +66,22 @@ function resolve(
return value;
}
-function recipe(value: SurfaceResolvedReference, label: string): SvsRecipe {
+const visualAppearancePropertyNames = [
+ ...contentFitPropertyNames,
+ ...mediaAppearanceKeys.sample.filter((name) => name !== "playback" && name !== "trim-start" && name !== "trim-end"),
+ ...mediaAppearanceKeys.frame.filter((name) => name !== "stack-order"),
+] as readonly string[];
+const visualMotionPropertyNames = mediaAppearanceKeys.motion.filter(
+ (name) => name !== "enter-origin" && name !== "exit-origin",
+);
+
+function recipe(value: SurfaceResolvedReference, label: string, allowed: readonly string[]): SvsRecipe {
if (!sameType(value.type, svsRecipeType) || value.record?.value.kind !== "inline") {
throw new Error(`${label} must be an authored SVS Recipe`);
}
const result = value.record.value.value as unknown as SvsRecipe;
- const unknown = Object.keys(result.properties).filter((name) => !contentFitPropertyNames.includes(
- name as (typeof contentFitPropertyNames)[number],
- ));
- if (unknown.length > 0) throw new Error(`${label} only accepts spatial fit properties; found ${unknown.join(", ")}`);
+ const unknown = Object.keys(result.properties).filter((name) => !allowed.includes(name));
+ if (unknown.length > 0) throw new Error(`${label} has unsupported properties ${unknown.join(", ")}`);
return result;
}
@@ -83,16 +99,37 @@ function takes(element: StructuredElement): StructuredElement[] {
return result;
}
+function samplingMotion(element: StructuredElement): MediaSamplingMotion | undefined {
+ const keyframes = element.children.flatMap((child) => {
+ if (child.kind === "text") {
+ if (child.value.trim()) throw new Error(`${element.name} accepts only Sampling children`);
+ return [];
+ }
+ if (localName(child.name) !== "Sampling") {
+ throw new Error(`${element.name} accepts only Sampling children`);
+ }
+ return [decodeMediaSamplingKeyframe(child)];
+ });
+ return keyframes.length === 0 ? undefined : { keyframes };
+}
+
export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- allowedAttributes(element, ["id", "visual-frame", "visual-appearance", "visual-z"], []);
+ allowedAttributes(element, ["id", "visual-frame", "visual-appearance", "visual-z"], ["visual-motion"]);
const id = stringAttribute(element, "id");
const baseFrame = resolve(element, "visual-frame", spatialTypes.frame, resolveReference);
const baseAppearanceReference = resolve(element, "visual-appearance", svsRecipeType, resolveReference);
const baseAppearance = recipe(
baseAppearanceReference,
`${element.name}.visual-appearance`,
+ visualAppearancePropertyNames,
);
const baseZ = integerAttribute(element, "visual-z");
+ const baseMotionReference = element.attributes["visual-motion"] === undefined
+ ? undefined
+ : resolve(element, "visual-motion", svsRecipeType, resolveReference);
+ const baseMotion = baseMotionReference === undefined
+ ? undefined
+ : recipe(baseMotionReference, `${element.name}.visual-motion`, visualMotionPropertyNames);
const frameInputNames = new Map<string, string>();
const frameInputName = (value: SurfaceResolvedReference, fallback: string): string => {
const key = JSON.stringify(value.ref);
@@ -109,15 +146,16 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, re
fitInputNames.set(key, fallback);
return fallback;
};
- const visualSpecInputNames = new Map<number, string>();
- const visualSpecInputName = (stackingOrder: number, fallback: string): string => {
- const previous = visualSpecInputNames.get(stackingOrder);
+ const visualSpecInputNames = new Map<string, string>();
+ const visualSpecInputName = (spec: ReturnType<typeof sealSpeechTrackVisualSpec>, fallback: string): string => {
+ const key = JSON.stringify(spec);
+ const previous = visualSpecInputNames.get(key);
if (previous !== undefined) return previous;
- visualSpecInputNames.set(stackingOrder, fallback);
+ visualSpecInputNames.set(key, fallback);
return fallback;
};
const declaredTakes = takes(element).map((take, index) => {
- allowedAttributes(take, ["source"], ["frame", "appearance", "z"]);
+ allowedAttributes(take, ["source"], ["frame", "appearance", "motion", "z"]);
const suffix = String(index + 1).padStart(4, "0");
const effectiveFrame = take.attributes.frame === undefined
? baseFrame
@@ -127,15 +165,31 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, re
: resolve(take, "appearance", svsRecipeType, resolveReference);
const effectiveAppearance = take.attributes.appearance === undefined
? baseAppearance
- : recipe(effectiveAppearanceReference, `${take.name}.appearance`);
+ : recipe(effectiveAppearanceReference, `${take.name}.appearance`, visualAppearancePropertyNames);
const effectiveZ = take.attributes.z === undefined ? baseZ : integerAttribute(take, "z");
+ const effectiveMotionReference = take.attributes.motion === undefined
+ ? baseMotionReference
+ : resolve(take, "motion", svsRecipeType, resolveReference);
+ const effectiveMotion = take.attributes.motion === undefined
+ ? baseMotion
+ : recipe(effectiveMotionReference!, `${take.name}.motion`, visualMotionPropertyNames);
+ const effectiveSamplingMotion = samplingMotion(take);
+ const framePaint = decodeMediaFramePaint(effectiveAppearance, `${id}.take.${suffix}.frame-paint`);
+ const visualSpec = sealSpeechTrackVisualSpec({
+ stackingOrder: effectiveZ,
+ presentation: decodeMediaPresentation(effectiveAppearance),
+ sampleAppearance: decodeMediaSampleAppearance(effectiveAppearance),
+ motion: decodeMediaMotion(effectiveMotion),
+ ...(framePaint === undefined ? {} : { framePaint }),
+ ...(effectiveSamplingMotion === undefined ? {} : { samplingMotion: effectiveSamplingMotion }),
+ });
const visual = {
frameName: frameInputName(effectiveFrame, take.attributes.frame === undefined ? "visual-base-frame" : `take-${suffix}-frame`),
fitName: fitInputName(effectiveAppearanceReference, take.attributes.appearance === undefined ? "visual-base-fit" : `take-${suffix}-fit`),
- visualSpecName: visualSpecInputName(effectiveZ, take.attributes.z === undefined ? "visual-base-spec" : `take-${suffix}-visual-spec`),
+ visualSpecName: visualSpecInputName(visualSpec, `take-${suffix}-visual-spec`),
frame: effectiveFrame,
appearance: effectiveAppearance,
- stackingOrder: effectiveZ,
+ spec: visualSpec,
};
return {
suffix,
@@ -179,10 +233,7 @@ export const decodeSpeechTrackSurface: StructuredSurfaceHandler = ({ element, re
records.push({
id: `${id}.${take.visual.visualSpecName}`,
type: speechTrackTypes.visualSpec,
- value: { kind: "inline" as const, value: sealSpeechTrackVisualSpec({
-
- stackingOrder: take.visual.stackingOrder,
- }) },
+ value: { kind: "inline" as const, value: take.visual.spec },
range: take.range,
});
}
diff --git a/packages/speech-track/src/types.ts b/packages/speech-track/src/types.ts
index 658b3ad1..567d684a 100644
--- a/packages/speech-track/src/types.ts
+++ b/packages/speech-track/src/types.ts
@@ -1,4 +1,11 @@
import type { ContentFit, SpatialFrame } from "@hypit/spatial";
+import type {
+ MediaFramePresentation,
+ MediaLifecycleMotion,
+ MediaPaintLayerSpec,
+ MediaSampleAppearance,
+ MediaSamplingMotion,
+} from "@hypit/media-track";
/** Author-owned policy for one ordered Speech Track. */
export type SpeechTrackHeader = {
@@ -10,12 +17,17 @@ export type SpeechTrackTake = {
readonly visual?: {
readonly frame: SpatialFrame;
readonly fit: ContentFit;
- readonly stackingOrder: number;
+ readonly spec: SpeechTrackVisualSpec;
};
};
export type SpeechTrackVisualSpec = {
readonly stackingOrder: number;
+ readonly presentation: MediaFramePresentation;
+ readonly sampleAppearance: MediaSampleAppearance;
+ readonly motion: MediaLifecycleMotion;
+ readonly framePaint?: MediaPaintLayerSpec;
+ readonly samplingMotion?: MediaSamplingMotion;
};
/**
diff --git a/packages/speech-track/test/program.test.ts b/packages/speech-track/test/program.test.ts
index 218ff5b5..f4a26243 100644
--- a/packages/speech-track/test/program.test.ts
+++ b/packages/speech-track/test/program.test.ts
@@ -52,7 +52,24 @@ function semantic(id: string, visual: boolean): SemanticTake {
test("one Track assembles a continuous SemanticTrack and only visual Takes become presents", () => {
const header = sealSpeechTrackHeader({ id: "speech" });
- const visualSpec = sealSpeechTrackVisualSpec({ stackingOrder: 30 });
+ const visualSpec = sealSpeechTrackVisualSpec({
+ stackingOrder: 30,
+ presentation: {
+ clip: { kind: "rounded", radiusPx: 320 },
+ padding: { topPx: 0, rightPx: 0, bottomPx: 0, leftPx: 0 },
+ border: { widthPx: 4, style: "solid", color: "#ffffff" },
+ shadows: [],
+ },
+ sampleAppearance: {
+ opacity: 1,
+ filter: { blurPx: 0, brightness: 1, contrast: 1, saturation: 1 },
+ },
+ motion: { sustain: [] },
+ samplingMotion: { keyframes: [
+ { atProgress: 0, zoom: 1, offsetX: 0, offsetY: 0, rotationDeg: 0 },
+ { atProgress: 1, zoom: 1.08, offsetX: 0, offsetY: -18, rotationDeg: 0 },
+ ] },
+ });
let set = createSpeechTrackSet();
set = appendSpeechTrackTake(set, semantic("voiceover", false), frame, fit, visualSpec);
set = appendSpeechTrackTake(set, semantic("answer", true), frame, fit, visualSpec);
@@ -64,4 +81,8 @@ test("one Track assembles a continuous SemanticTrack and only visual Takes becom
assert.equal(visualTrack.presents.length, 1);
assert.equal(visualTrack.presents[0]?.id, "speech:visual:answer");
assert.deepEqual(visualTrack.presents[0]?.span, { startFrame: 30, endFrameExclusive: 60 });
+ const styles = visualTrack.presents[0]?.elements.flatMap((element) => element.style) ?? [];
+ assert.equal(styles.find((style) => style.name === "border-radius")?.value, "320px");
+ assert.equal(styles.find((style) => style.name === "border")?.value, "4px solid #ffffff");
+ assert.equal(visualTrack.presents[0]?.elements.some((element) => element.animation !== undefined), true);
});
diff --git a/packages/speech-track/test/surface.test.ts b/packages/speech-track/test/surface.test.ts
index 106d33a8..b7451f5f 100644
--- a/packages/speech-track/test/surface.test.ts
+++ b/packages/speech-track/test/surface.test.ts
@@ -56,7 +56,10 @@ function resolved(path: string): SurfaceResolvedReference | undefined {
ref: { kind: "record", id: path },
type: svsRecipeType,
record: { id: path, type: svsRecipeType, value: { kind: "inline", value: {
- path: "speech.base", properties: { fit: "cover" },
+ path: "speech.base", properties: {
+ fit: "cover", clip: "rounded", radius: 360,
+ "border-width": 4, "border-color": "#ffffff",
+ },
} } },
};
return undefined;
@@ -76,7 +79,17 @@ test("Speech Track accepts only already-semantic Segment Takes", async () => {
kind: "element",
name: "speech:Take",
attributes: { source: { kind: "reference", path: "opening.take" } },
- children: [],
+ children: [
+ {
+ kind: "element", name: "speech:Sampling",
+ attributes: { at: "start", zoom: "1" }, children: [], range: { start: 30, end: 40 },
+ },
+ {
+ kind: "element", name: "speech:Sampling",
+ attributes: { at: "end", zoom: "1.08", y: "-18", easing: "ease-out" },
+ children: [], range: { start: 41, end: 55 },
+ },
+ ],
range: { start: 20, end: 60 },
}],
range: { start: 0, end: 70 },
@@ -93,4 +106,17 @@ test("Speech Track accepts only already-semantic Segment Takes", async () => {
audio: "speech.audio",
});
assert.equal(output.exports?.includes("speech.audio") ?? false, true);
+ const visualSpec = output.records.find((record) => record.type.name === "SpeechTrackVisualSpec");
+ assert.equal(visualSpec?.value.kind, "inline");
+ if (visualSpec?.value.kind !== "inline") throw new Error("Speech Track visual spec was not materialized");
+ const value = visualSpec.value.value as unknown as {
+ presentation: { clip: { kind: string; radiusPx?: number }; border?: { widthPx: number } };
+ samplingMotion?: { keyframes: readonly { atProgress: number; zoom: number; offsetY: number }[] };
+ };
+ assert.deepEqual(value.presentation.clip, { kind: "rounded", radiusPx: 360 });
+ assert.equal(value.presentation.border?.widthPx, 4);
+ assert.deepEqual(value.samplingMotion?.keyframes.map(({ atProgress, zoom, offsetY }) => ({ atProgress, zoom, offsetY })), [
+ { atProgress: 0, zoom: 1, offsetY: 0 },
+ { atProgress: 1, zoom: 1.08, offsetY: -18 },
+ ]);
});