Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat(media-pipeline): StillVideo spreads several pictures over one literal duration

kashorind6ab873bb111
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index 47d2c97f..a51fd08a 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -483,9 +483,9 @@ function extractFrameNeed(value: CanonicalValue): ExtractFrameNeed {
function renderStillVideoNeed(value: CanonicalValue): RenderStillVideoNeed {
const item = object(value, "RenderStillVideoNeed") as unknown as RenderStillVideoNeed;
- assert(item.source?.kind === "blob" && item.source.mediaType.startsWith("image/"),
- "RenderStillVideoNeed source must be an image Artifact");
verifyStillVideoRequest(item.request);
+ assert(item.request.segments.every((segment) => segment.source?.kind === "blob" && segment.source.mediaType.startsWith("image/")),
+ "RenderStillVideoNeed needs an image Artifact for every segment");
return item;
}
@@ -729,7 +729,11 @@ export async function executeNormalizeMedia(
}
}
-/** Encode one authored image into an exact silent CFR video. */
+/**
+ * Encode authored images into an exact silent CFR video. One picture is held for the whole frame
+ * count; several are each held for their planned segment, fitted into the first picture's frame
+ * and letterboxed on black, then concatenated in order.
+ */
export async function executeRenderStillVideo(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
@@ -737,22 +741,54 @@ export async function executeRenderStillVideo(
const need = renderStillVideoNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-still-"));
try {
- const input = join(work, "source.image");
const output = join(work, "still.mp4");
- await stageArtifact(env, need.source, input);
- const fps = `${need.request.frameRate.numerator}/${need.request.frameRate.denominator}`;
- const filter = [
+ const staged = new Map<string, string>();
+ const inputs: string[] = [];
+ for (const segment of need.request.segments) {
+ const source = segment.source!;
+ let path = staged.get(source.resource);
+ if (path === undefined) {
+ path = join(work, `picture-${staged.size}.image`);
+ await stageArtifact(env, source, path);
+ staged.set(source.resource, path);
+ }
+ inputs.push(path);
+ }
+ const { numerator, denominator } = need.request.frameRate;
+ const fps = `${numerator}/${denominator}`;
+ const hold = (frames: number): string => [
"select=eq(n\\,0)",
"loop=loop=-1:size=1:start=0",
- `trim=start_frame=0:end_frame=${need.request.frameCount}`,
- `setpts=N*${need.request.frameRate.denominator}/(${need.request.frameRate.numerator}*TB)`,
- "pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black",
- "setsar=1",
+ `trim=start_frame=0:end_frame=${frames}`,
+ `setpts=N*${denominator}/(${numerator}*TB)`,
].join(",");
+ let argv: string[];
+ if (need.request.segments.length === 1) {
+ const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`;
+ argv = ["-y", "-i", inputs[0]!, "-map", "0:v:0", "-an", "-vf", filter];
+ } else {
+ const first = await outputInspection({
+ path: inputs[0]!,
+ mediaType: need.request.segments[0]!.source!.mediaType,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ const picture = first.streams.find((item): item is MediaVideoStream => item.kind === "video");
+ assert(picture !== undefined, "Still video first picture has no decodable image");
+ const width = Math.ceil(picture.width / 2) * 2;
+ const height = Math.ceil(picture.height / 2) * 2;
+ const chains = need.request.segments.map((segment, index) =>
+ `[${index}:v]${hold(segment.endFrameExclusive - segment.startFrame)},`
+ + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p[s${index}]`);
+ const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[v]`;
+ argv = ["-y", ...inputs.flatMap((path) => ["-i", path]), "-filter_complex", `${chains.join(";")};${concat}`, "-map", "[v]", "-an"];
+ }
await runProcess({
executable: env.ffmpegPath,
argv: [
- "-y", "-i", input, "-map", "0:v:0", "-an", "-vf", filter,
+ ...argv,
"-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr",
"-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
"-movflags", "+faststart", output,
diff --git a/packages/media-pipeline/README.md b/packages/media-pipeline/README.md
index fba8f51a..5e91acad 100644
--- a/packages/media-pipeline/README.md
+++ b/packages/media-pipeline/README.md
@@ -52,10 +52,24 @@ A still image becomes ordinary video before it enters that waist:
video="primary-moving" audio="none" span-authority="video" clock={clock}/>
```
-`StillVideo` returns a silent MP4 `BlobArtifact`, not `SynchronizedMedia`. Encoding its first decoded
-image frame is a `render-still-video` Need; inspection and normalization remain the same explicit
-steps used by imported or generated moving video. The Blob can therefore serve B-roll through
-Normalize, or continue into either estimated or measured A-roll semantics afterward.
+Several images spread over one literal duration the same way, each held for its share of the frames:
+
+```svml
+<media:StillVideo id="kitchen-stills" duration="6" clock={clock}>
+ <media:Still source={counter.image}/>
+ <media:Still source={basil.image} weight="2"/>
+ <media:Still source={board.image}/>
+</media:StillVideo>
+```
+
+`StillVideo` returns a silent MP4 `BlobArtifact`, not `SynchronizedMedia`. The Surface publishes the
+duration and the weights as Records; `plan-still-video` divides the whole frame count among the
+pictures (every picture holds at least one frame, the rest go by weight with leftovers to the largest
+remainders, earlier first), `bind-still-video-source` attaches each picture in authored order, and
+encoding is one `render-still-video` Need. Pictures of different sizes are fitted into the first one's
+frame and letterboxed. Inspection and normalization remain the same explicit steps used by imported
+or generated moving video, so the Blob can serve B-roll through Normalize, or continue into either
+estimated or measured A-roll semantics afterward.
Four ordinary author operations reuse that same inspection/execution boundary:
diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.ts
index 2d8bab94..48c14038 100644
--- a/packages/media-pipeline/src/component.ts
+++ b/packages/media-pipeline/src/component.ts
@@ -22,7 +22,10 @@ import {
import {
selectAudioStream,
selectVideoStream,
+ bindStillVideoSource,
+ planStillVideoSegments,
sealStillVideoRequest,
+ verifyStillVideoLayout,
verifyAudioExtractionRequest,
verifyFrameExtractionRequest,
verifyMediaTransformProgram,
@@ -40,6 +43,7 @@ import type {
ProjectSpeechEvidenceAudioNeed,
RenderAudioNeed,
RenderStillVideoNeed,
+ StillVideoLayout,
StillVideoRequest,
TransformMediaNeed,
} from "./types.js";
@@ -86,6 +90,12 @@ export const mediaPipelineComponent = {
verifyFrameExtractionRequest(inline(value, "FrameExtractionRequest"));
},
},
+ {
+ type: mediaPipelineTypes.stillVideoLayout,
+ handler: ({ value }) => {
+ verifyStillVideoLayout(inline(value, "StillVideoLayout"));
+ },
+ },
{
type: mediaPipelineTypes.stillVideoRequest,
handler: ({ value }) => {
@@ -198,6 +208,8 @@ export const mediaPipelineComponent = {
const clock = inline(inputs.clock!.value, "ProgramClock") as unknown as ProgramClock;
assertSpeechDurationIdentity(duration);
assertProgramClockIdentity(clock);
+ const layout = inline(inputs.layout!.value, "StillVideoLayout") as unknown as StillVideoLayout;
+ verifyStillVideoLayout(layout);
const frames = Math.round(duration * clock.frameRate.numerator / clock.frameRate.denominator);
if (!Number.isSafeInteger(frames) || frames < 1) {
throw new Error("Still video duration does not produce a positive safe frame count");
@@ -206,18 +218,29 @@ export const mediaPipelineComponent = {
frameRate: clock.frameRate,
frameCount: frames,
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: planStillVideoSegments(frames, layout.weights),
});
return { outputs: { request: { kind: "inline", value: canonicalize(request) } }, needs: {} };
},
},
{
- producer: mediaPipelineProducers.renderStill,
+ producer: mediaPipelineProducers.bindStill,
handler: ({ inputs }) => {
+ const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;
const source = blob(inputs.source!.value, "Still video source");
+ const bound = bindStillVideoSource(request, source);
+ return { outputs: { request: { kind: "inline", value: canonicalize(bound) } }, needs: {} };
+ },
+ },
+ {
+ producer: mediaPipelineProducers.renderStill,
+ handler: ({ inputs }) => {
const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;
- if (!source.mediaType.startsWith("image/")) throw new Error("Still video source must be an image Artifact");
verifyStillVideoRequest(request);
- const need: RenderStillVideoNeed = { source, request };
+ if (request.segments.some((segment) => segment.source === undefined)) {
+ throw new Error("Still video has a segment without a picture");
+ }
+ const need: RenderStillVideoNeed = { request };
return { outputs: {}, needs: { video: canonicalize(need) } };
},
},
diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.ts
index c4a927b1..18651440 100644
--- a/packages/media-pipeline/src/fragment.ts
+++ b/packages/media-pipeline/src/fragment.ts
@@ -121,29 +121,47 @@ export const extractFrameFragment = sealGraphFragment({
}],
});
-export const stillVideoFragment = sealGraphFragment({
- inputs: [
- { name: "source", type: artifactTypes.blob },
- { name: "duration", type: speechTypes.duration },
- { name: "clock", type: programSpaceTypes.clock },
- ],
- operations: [
- {
- id: "plan",
- producer: mediaPipelineProducers.planStill,
- inputs: { duration: input("duration"), clock: input("clock") },
- result: { kind: "output", name: "request" },
- },
- {
- id: "render",
- producer: mediaPipelineProducers.renderStill,
- inputs: { source: input("source"), request: operation("plan") },
- result: { kind: "need", name: "video" },
- },
- ],
- exports: [{
- name: "video",
- type: artifactTypes.blob,
- root: operation("render"),
- }],
-});
+/**
+ * A still video over `count` pictures: plan the frame split, bind each picture to its segment in
+ * authored order, render once. The shape depends only on the count, so two elements with the same
+ * number of pictures share one Fragment identity.
+ */
+export function createStillVideoFragment(count: number) {
+ if (!Number.isSafeInteger(count) || count < 1) throw new Error("A still video needs at least one picture");
+ const sources = Array.from({ length: count }, (_, index) => `source-${index}`);
+ return sealGraphFragment({
+ inputs: [
+ { name: "duration", type: speechTypes.duration },
+ { name: "clock", type: programSpaceTypes.clock },
+ { name: "layout", type: mediaPipelineTypes.stillVideoLayout },
+ ...sources.map((name) => ({ name, type: artifactTypes.blob })),
+ ],
+ operations: [
+ {
+ id: "plan",
+ producer: mediaPipelineProducers.planStill,
+ inputs: { duration: input("duration"), clock: input("clock"), layout: input("layout") },
+ result: { kind: "output", name: "request" },
+ },
+ ...sources.map((name, index) => ({
+ id: `bind-${index}`,
+ producer: mediaPipelineProducers.bindStill,
+ inputs: { request: operation(index === 0 ? "plan" : `bind-${index - 1}`), source: input(name) },
+ result: { kind: "output" as const, name: "request" },
+ })),
+ {
+ id: "render",
+ producer: mediaPipelineProducers.renderStill,
+ inputs: { request: operation(`bind-${count - 1}`) },
+ result: { kind: "need", name: "video" },
+ },
+ ],
+ exports: [{
+ name: "video",
+ type: artifactTypes.blob,
+ root: operation("render"),
+ }],
+ });
+}
+
+export const stillVideoFragment = createStillVideoFragment(1);
diff --git a/packages/media-pipeline/src/index.ts b/packages/media-pipeline/src/index.ts
index bc1d13ba..5439410d 100644
--- a/packages/media-pipeline/src/index.ts
+++ b/packages/media-pipeline/src/index.ts
@@ -7,6 +7,7 @@ export {
extractFrameFragment,
synchronizedMediaFragment,
stillVideoFragment,
+ createStillVideoFragment,
transformMediaFragment,
} from "./fragment.js";
export {
@@ -30,12 +31,16 @@ export {
sealAudioExtractionRequest,
sealFrameExtractionRequest,
sealMediaTransformProgram,
+ sealStillVideoLayout,
sealStillVideoRequest,
+ planStillVideoSegments,
+ bindStillVideoSource,
selectAudioStream,
selectVideoStream,
verifyAudioExtractionRequest,
verifyFrameExtractionRequest,
verifyMediaTransformProgram,
+ verifyStillVideoLayout,
verifyStillVideoRequest,
} from "./operations.js";
export type * from "./types.js";
diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.ts
index 1037d07c..88ca2df3 100644
--- a/packages/media-pipeline/src/manifest.ts
+++ b/packages/media-pipeline/src/manifest.ts
@@ -21,6 +21,7 @@ export const mediaPipelineTypes = {
audioExtractionRequest: { module: mediaPipelineModuleRef, name: "AudioExtractionRequest" },
frameExtractionRequest: { module: mediaPipelineModuleRef, name: "FrameExtractionRequest" },
stillVideoRequest: { module: mediaPipelineModuleRef, name: "StillVideoRequest" },
+ stillVideoLayout: { module: mediaPipelineModuleRef, name: "StillVideoLayout" },
} satisfies Record<string, TypeRef>;
export const mediaPipelineCapabilities = {
inspect: { module: mediaPipelineModuleRef, name: "inspect-media" },
@@ -42,6 +43,7 @@ export const mediaPipelineProducers = {
extractFrame: { module: mediaPipelineModuleRef, name: "request-frame-extraction" },
planStill: { module: mediaPipelineModuleRef, name: "plan-still-video" },
renderStill: { module: mediaPipelineModuleRef, name: "request-still-video" },
+ bindStill: { module: mediaPipelineModuleRef, name: "bind-still-video-source" },
projectSpeechEvidenceAudio: { module: mediaPipelineModuleRef, name: "request-speech-evidence-audio" },
planAudio: { module: mediaPipelineModuleRef, name: "compile-audio-program" },
renderAudio: { module: mediaPipelineModuleRef, name: "request-audio-render" },
@@ -168,6 +170,29 @@ export const stillVideoRequestSchema: ValueSchema = {
pixelFormat: { schema: { kind: "literal", value: "yuv420p" } },
},
} },
+ segments: { schema: { kind: "array", items: {
+ kind: "object",
+ fields: {
+ startFrame: { schema: { kind: "number", integer: true, minimum: 0 } },
+ endFrameExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },
+ source: { optional: true, schema: {
+ kind: "object",
+ fields: {
+ kind: { schema: { kind: "literal", value: "blob" } },
+ resource: { schema: { kind: "string", minLength: 5, maxLength: 256 } },
+ size: { schema: { kind: "number", integer: true, minimum: 0 } },
+ mediaType: { schema: { kind: "string", minLength: 6 } },
+ },
+ } },
+ },
+ } } },
+ },
+};
+
+export const stillVideoLayoutSchema: ValueSchema = {
+ kind: "object",
+ fields: {
+ weights: { schema: { kind: "array", items: { kind: "number", minimum: 0 } } },
},
};
@@ -265,24 +290,39 @@ export const mediaPipelineMarkupSurfaces = [
},
{
name: "still-video", tag: "StillVideo", mode: "structured",
- outputs: [speechTypes.duration, mediaPipelineTypes.stillVideoRequest, artifactTypes.blob],
+ outputs: [speechTypes.duration, mediaPipelineTypes.stillVideoLayout, mediaPipelineTypes.stillVideoRequest, artifactTypes.blob],
vocabulary: {
- summary: "Encodes one authored image as an ordinary silent MP4 Blob on an explicit duration and frame clock.",
+ summary: "Spreads one or more authored images over an explicit duration as an ordinary silent MP4 Blob on a frame clock.",
attributes: [
{ name: "id", kind: "identifier", required: true,
summary: "Names the still-video operation and the MP4 Artifact it publishes." },
- { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],
- summary: "Selects the authored image whose first decoded frame is held for the full video." },
+ { name: "source", kind: "reference", required: false, accepts: [artifactTypes.blob],
+ summary: "Selects the one authored image held for the full video; write Still children instead for several images." },
{ name: "duration", kind: "literal", required: true,
summary: "Sets the video's length in seconds, such as 6 or 2.5s; the author's decision, measured beforehand with hypit measure." },
{ name: "clock", kind: "reference", required: true, accepts: [programSpaceTypes.clock],
summary: "Selects the frame clock used by the generated MP4." },
],
+ children: [
+ { tag: "Still", cardinality: "many",
+ summary: "One image in authored order; the duration is divided among the Still children by weight, equally unless weights say otherwise.",
+ attributes: [
+ { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],
+ summary: "Selects the authored image whose first decoded frame is held for this share." },
+ { name: "weight", kind: "literal", required: false,
+ summary: "Sets this image's share of the duration relative to its siblings, such as 2; every Still weighs 1 unless written." },
+ ] },
+ ],
ports: [{ name: "video", type: artifactTypes.blob,
summary: "The ordinary silent MP4 Artifact, addressed as `<id>.video`." }],
- example: `<media:StillVideo id="opening-still" source={opening-head}
- duration={opening-duration.duration} clock={clock}/>`,
+ example: `<media:StillVideo id="kitchen-stills" duration="6" clock={clock}>
+ <media:Still source={counter}/>
+ <media:Still source={basil} weight="2"/>
+ <media:Still source={board}/>
+</media:StillVideo>`,
notes: [
+ "Write exactly one of source or Still children. Frames are whole: each image gets the floor of its share and the leftover frames go to the largest remainders, so the split is deterministic and every image holds at least one frame.",
+ "Images of different sizes are fitted into the first image's frame, letterboxed on black.",
"The result is a normal video Blob, not SynchronizedMedia and not a SemanticTake.",
"Use Normalize afterward exactly as for generated or imported moving video.",
"Encoding is a render-still-video Need fulfilled by the selected media Provider; this Surface never invokes FFmpeg itself.",
@@ -425,6 +465,9 @@ export const mediaPipelineManifest: ModuleManifest = {
{
name: mediaPipelineTypes.stillVideoRequest.name,
},
+ {
+ name: mediaPipelineTypes.stillVideoLayout.name,
+ },
],
capabilities: [
{ name: mediaPipelineCapabilities.inspect.name, returns: mediaTypes.inspection },
@@ -521,14 +564,23 @@ export const mediaPipelineManifest: ModuleManifest = {
inputs: [
{ name: "duration", type: speechTypes.duration },
{ name: "clock", type: programSpaceTypes.clock },
+ { name: "layout", type: mediaPipelineTypes.stillVideoLayout },
],
outputs: [{ name: "request", type: mediaPipelineTypes.stillVideoRequest }],
needs: [],
},
{
- name: mediaPipelineProducers.renderStill.name,
+ name: mediaPipelineProducers.bindStill.name,
inputs: [
+ { name: "request", type: mediaPipelineTypes.stillVideoRequest },
{ name: "source", type: artifactTypes.blob },
+ ],
+ outputs: [{ name: "request", type: mediaPipelineTypes.stillVideoRequest }],
+ needs: [],
+ },
+ {
+ name: mediaPipelineProducers.renderStill.name,
+ inputs: [
{ name: "request", type: mediaPipelineTypes.stillVideoRequest },
],
outputs: [],
diff --git a/packages/media-pipeline/src/operations.ts b/packages/media-pipeline/src/operations.ts
index ae8d6554..ad14e19c 100644
--- a/packages/media-pipeline/src/operations.ts
+++ b/packages/media-pipeline/src/operations.ts
@@ -1,6 +1,7 @@
import { verifyMediaInspection } from "@hypit/media";
import type { MediaAudioStream, MediaInspection, MediaVideoStream } from "@hypit/media";
import { canonicalize } from "@hypit/protocol";
+import type { BlobRef } from "@hypit/protocol";
import type {
AudioExtractionRequest,
@@ -8,7 +9,9 @@ import type {
MediaAudioSelector,
MediaTransformProgram,
MediaVideoSelector,
+ StillVideoLayout,
StillVideoRequest,
+ StillVideoSegment,
} from "./types.js";
function assert(condition: unknown, message: string): asserts condition {
@@ -114,6 +117,51 @@ export function sealFrameExtractionRequest(value: FrameExtractionRequest): Frame
return result;
}
+export function verifyStillVideoLayout(value: unknown): asserts value is StillVideoLayout {
+ const layout = object(value, "StillVideoLayout");
+ assert(Array.isArray(layout.weights) && layout.weights.length > 0, "StillVideoLayout needs at least one weight");
+ for (const weight of layout.weights) {
+ assert(typeof weight === "number" && Number.isFinite(weight) && weight > 0, "StillVideoLayout weights must be positive");
+ }
+}
+
+export function sealStillVideoLayout(value: StillVideoLayout): StillVideoLayout {
+ const result = canonicalize(value) as unknown as StillVideoLayout;
+ verifyStillVideoLayout(result);
+ return result;
+}
+
+/**
+ * Spread `frameCount` whole frames over the weights. Every picture first holds one frame; the rest
+ * are shared by weight, each picture taking the floor of its share and the frames left over going
+ * one each to the largest fractional remainders, earlier picture first on a tie. Deterministic, and
+ * refused when there are fewer frames than pictures.
+ */
+export function planStillVideoSegments(frameCount: number, weights: readonly number[]): readonly StillVideoSegment[] {
+ assert(Number.isSafeInteger(frameCount) && frameCount > 0, "Still video frame count must be positive");
+ assert(weights.length > 0, "Still video needs at least one picture");
+ assert(frameCount >= weights.length, `Still video has ${frameCount} frames for ${weights.length} pictures; each picture needs at least one`);
+ const total = weights.reduce((sum, weight) => sum + weight, 0);
+ const shared = frameCount - weights.length;
+ const exact = weights.map((weight) => shared * weight / total);
+ const floors = exact.map((share) => 1 + Math.floor(share));
+ let remaining = frameCount - floors.reduce((sum, item) => sum + item, 0);
+ const order = exact.map((share, index) => ({ index, fraction: share - Math.floor(share) }))
+ .sort((left, right) => right.fraction - left.fraction || left.index - right.index);
+ for (const { index } of order) {
+ if (remaining === 0) break;
+ floors[index] = floors[index]! + 1;
+ remaining -= 1;
+ }
+ const segments: StillVideoSegment[] = [];
+ let start = 0;
+ for (const frames of floors) {
+ segments.push({ startFrame: start, endFrameExclusive: start + frames });
+ start += frames;
+ }
+ return segments;
+}
+
export function verifyStillVideoRequest(value: unknown): asserts value is StillVideoRequest {
const request = object(value, "StillVideoRequest");
const frameRate = object(request.frameRate, "StillVideoRequest.frameRate");
@@ -125,6 +173,21 @@ export function verifyStillVideoRequest(value: unknown): asserts value is StillV
const output = object(request.output, "StillVideoRequest.output");
assert(output.container === "mp4" && output.codec === "h264" && output.pixelFormat === "yuv420p",
"StillVideoRequest output profile is unsupported");
+ assert(Array.isArray(request.segments) && request.segments.length > 0, "StillVideoRequest needs at least one segment");
+ let cursor = 0;
+ for (const item of request.segments) {
+ const segment = object(item, "StillVideoRequest.segments[]");
+ assert(segment.startFrame === cursor, "StillVideoRequest segments must be contiguous from frame 0");
+ assert(Number.isSafeInteger(segment.endFrameExclusive) && (segment.endFrameExclusive as number) > cursor,
+ "StillVideoRequest segments must each hold at least one frame");
+ cursor = segment.endFrameExclusive as number;
+ if (segment.source !== undefined) {
+ const source = object(segment.source, "StillVideoRequest.segments[].source");
+ assert(source.kind === "blob" && typeof source.mediaType === "string" && source.mediaType.startsWith("image/"),
+ "StillVideoRequest segment source must be an image Artifact");
+ }
+ }
+ assert(cursor === request.frameCount, "StillVideoRequest segments must cover exactly frameCount frames");
}
export function sealStillVideoRequest(value: StillVideoRequest): StillVideoRequest {
@@ -133,6 +196,18 @@ export function sealStillVideoRequest(value: StillVideoRequest): StillVideoReque
return result;
}
+/** Attach one picture to the first segment that has none; the pictures arrive in authored order. */
+export function bindStillVideoSource(request: StillVideoRequest, source: BlobRef): StillVideoRequest {
+ verifyStillVideoRequest(request);
+ assert(source.kind === "blob" && source.mediaType.startsWith("image/"), "Still video source must be an image Artifact");
+ const index = request.segments.findIndex((segment) => segment.source === undefined);
+ assert(index >= 0, "Still video already has a picture for every segment");
+ return sealStillVideoRequest({
+ ...request,
+ segments: request.segments.map((segment, at) => at === index ? { ...segment, source } : segment),
+ });
+}
+
function uniqueDefault<T extends { readonly disposition: { readonly default: boolean }; readonly index: number }>(
values: readonly T[],
subject: string,
diff --git a/packages/media-pipeline/src/surface.ts b/packages/media-pipeline/src/surface.ts
index e3b0859b..c1e94e06 100644
--- a/packages/media-pipeline/src/surface.ts
+++ b/packages/media-pipeline/src/surface.ts
@@ -17,7 +17,7 @@ import { svsRecipeType, type SvsRecipe } from "@hypit/svs";
import {
extractAudioFragment,
extractFrameFragment,
- stillVideoFragment,
+ createStillVideoFragment,
synchronizedMediaFragment,
transformMediaFragment,
} from "./fragment.js";
@@ -228,36 +228,76 @@ export const decodeSynchronizedMediaSurface: StructuredSurfaceHandler = ({ eleme
};
};
+/** The pictures of a StillVideo in authored order, with their weights: one `source`, or Still children. */
+function stillPictures(
+ element: StructuredElement,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+): readonly { readonly source: SurfaceResolvedReference; readonly weight: number }[] {
+ const children = element.children.filter((child) => child.kind === "element" || child.value.trim().length > 0);
+ if (element.attributes.source !== undefined) {
+ if (children.length > 0) throw new Error(`${element.name} takes either source or Still children, not both.`);
+ return [{ source: ref(element.attributes.source, `${element.name}.source`, resolveReference), weight: 1 }];
+ }
+ const pictures = children.map((child) => {
+ if (child.kind !== "element" || localName(child.name) !== "Still") {
+ throw new Error(`${element.name} accepts only Still children.`);
+ }
+ exactAttributes(child, ["source"], ["weight"]);
+ empty(child);
+ const rawWeight = child.attributes.weight;
+ let weight = 1;
+ if (rawWeight !== undefined) {
+ if (typeof rawWeight !== "string") throw new Error(`${child.name}.weight must be text.`);
+ weight = Number(rawWeight);
+ if (!Number.isFinite(weight) || weight <= 0) throw new Error(`${child.name}.weight must be a positive number.`);
+ }
+ return { source: ref(child.attributes.source, `${child.name}.source`, resolveReference), weight };
+ });
+ if (pictures.length === 0) throw new Error(`${element.name} requires a source or at least one Still child.`);
+ return pictures;
+}
+
export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
- exactAttributes(element, ["id", "source", "duration", "clock"]);
- empty(element);
+ exactAttributes(element, ["id", "duration", "clock"], ["source"]);
const id = text(element, "id");
- const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);
+ const pictures = stillPictures(element, resolveReference);
// The duration is the author's decision, written here; it is published as an ordinary
// SpeechDuration Record so the fragment's plan step reads the same value a graph edge would carry.
const durationSec = seconds(text(element, "duration"), `${element.name}.duration`, false);
const durationId = `${id}.duration`;
+ const layoutId = `${id}.layout`;
const clock = typedReference(
element.attributes.clock,
`${element.name}.clock`,
programSpaceTypes.clock,
resolveReference,
);
+ const fragment = createStillVideoFragment(pictures.length);
return {
records: [{
id: durationId,
type: speechTypes.duration,
value: { kind: "inline", value: durationSec },
range: element.range,
+ }, {
+ id: layoutId,
+ type: mediaPipelineTypes.stillVideoLayout,
+ value: { kind: "inline", value: { weights: pictures.map((picture) => picture.weight) } },
+ range: element.range,
}],
components: [{
id,
- fragment: stillVideoFragment.id,
- inputs: { source: source.ref, duration: { kind: "record", id: durationId }, clock: clock.ref },
+ fragment: fragment.id,
+ inputs: {
+ duration: { kind: "record", id: durationId },
+ clock: clock.ref,
+ layout: { kind: "record", id: layoutId },
+ ...Object.fromEntries(pictures.map((picture, index) => [`source-${index}`, picture.source.ref])),
+ },
outputs: { video: `${id}.video` },
range: element.range,
}],
- fragments: [stillVideoFragment],
+ fragments: [fragment],
};
};
diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.ts
index 6773b4e6..4744025a 100644
--- a/packages/media-pipeline/src/types.ts
+++ b/packages/media-pipeline/src/types.ts
@@ -67,7 +67,19 @@ export type FrameExtractionRequest = {
readonly output: { readonly format: "png" };
};
-/** Exact silent video requested from one authored still image. */
+/** How the still video divides its literal duration among its pictures: one weight per picture, in order. */
+export type StillVideoLayout = {
+ readonly weights: readonly number[];
+};
+
+/** One picture's span of the still video, in frames; `source` is bound one picture at a time. */
+export type StillVideoSegment = {
+ readonly startFrame: number;
+ readonly endFrameExclusive: number;
+ readonly source?: BlobRef;
+};
+
+/** Exact silent video requested from one or more authored still images spread over a literal duration. */
export type StillVideoRequest = {
readonly frameRate: MediaRational;
readonly frameCount: number;
@@ -76,6 +88,7 @@ export type StillVideoRequest = {
readonly codec: "h264";
readonly pixelFormat: "yuv420p";
};
+ readonly segments: readonly StillVideoSegment[];
};
export type TransformMediaNeed = {
@@ -97,8 +110,8 @@ export type ExtractFrameNeed = {
readonly output: FrameExtractionRequest["output"];
};
+/** Every segment of the request carries its picture by the time the Need is made. */
export type RenderStillVideoNeed = {
- readonly source: BlobRef;
readonly request: StillVideoRequest;
};
diff --git a/packages/media-pipeline/test/manifest.test.ts b/packages/media-pipeline/test/manifest.test.ts
index 102b7a0b..c1467558 100644
--- a/packages/media-pipeline/test/manifest.test.ts
+++ b/packages/media-pipeline/test/manifest.test.ts
@@ -5,6 +5,7 @@ import { speechTypes } from "@hypit/speech";
import { svsRecipeType } from "@hypit/svs";
import assert from "node:assert/strict";
import test from "node:test";
+import { fixtureResource } from "../../../test/fixture-resource.js";
import {
mediaPipelineTypes,
@@ -13,6 +14,9 @@ import {
decodeExtractFrameSurface,
decodeTransformMediaSurface,
decodeStillVideoSurface,
+ createStillVideoFragment,
+ planStillVideoSegments,
+ bindStillVideoSource,
extractAudioFragment,
extractFrameFragment,
synchronizedMediaFragment,
@@ -145,5 +149,64 @@ test("StillVideo stops at an ordinary MP4 branch before Normalize", async () =>
id: "opening-still.duration", type: speechTypes.duration, value: { kind: "inline", value: 6 }, range,
}, "the literal duration is the author's, published as an ordinary SpeechDuration Record");
assert.deepEqual(output.components[0]?.inputs.duration, { kind: "record", id: "opening-still.duration" });
+ assert.deepEqual(output.components[0]?.inputs["source-0"], { kind: "record", id: "opening-head" });
+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1] } }, "one picture, one whole share");
assert.equal(stillVideoFragment.exports[0]?.type.name, artifactTypes.blob.name);
});
+
+test("StillVideo spreads several pictures over one literal duration by weight, in authored order", async () => {
+ const range = { source: "still.svml", start: 0, end: 1 };
+ const resolveReference = (path: string) => path === "clock"
+ ? { path, ref: { kind: "record" as const, id: path }, type: programSpaceTypes.clock }
+ : { path, ref: { kind: "record" as const, id: path }, type: artifactTypes.blob };
+ const child = (source: string, weight?: string) => ({
+ kind: "element" as const, name: "media:Still",
+ attributes: { source: { kind: "reference" as const, path: source }, ...(weight === undefined ? {} : { weight }) },
+ children: [], range,
+ });
+ const output = await decodeStillVideoSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element", name: "media:StillVideo",
+ attributes: { id: "kitchen", duration: "6", clock: { kind: "reference", path: "clock" } },
+ children: [child("counter"), child("basil", "2"), child("board")],
+ range,
+ },
+ resolveReference,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ });
+ assert.equal(output.fragments[0]?.id, createStillVideoFragment(3).id);
+ assert.notEqual(output.fragments[0]?.id, stillVideoFragment.id, "three pictures are a different shape from one");
+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1, 2, 1] } });
+ assert.deepEqual(
+ ["source-0", "source-1", "source-2"].map((name) => output.components[0]?.inputs[name]),
+ [{ kind: "record", id: "counter" }, { kind: "record", id: "basil" }, { kind: "record", id: "board" }],
+ );
+ await assert.rejects(async () => await decodeStillVideoSurface({
+ sourceName: range.source,
+ element: {
+ kind: "element", name: "media:StillVideo",
+ attributes: { id: "both", source: { kind: "reference", path: "counter" }, duration: "6", clock: { kind: "reference", path: "clock" } },
+ children: [child("basil")],
+ range,
+ },
+ resolveReference,
+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },
+ }), /either source or Still children/u);
+});
+
+test("still video frames are whole and every picture holds at least one", () => {
+ assert.deepEqual(planStillVideoSegments(180, [1, 2, 1]).map((item) => item.endFrameExclusive - item.startFrame), [45, 90, 45]);
+ assert.deepEqual(planStillVideoSegments(10, [1, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [4, 3, 3],
+ "the leftover frame goes to the earliest of equal remainders");
+ assert.deepEqual(planStillVideoSegments(10, [1, 1.5, 1]).map((item) => item.endFrameExclusive - item.startFrame), [3, 4, 3]);
+ assert.deepEqual(planStillVideoSegments(3, [100, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [1, 1, 1]);
+ assert.throws(() => planStillVideoSegments(2, [1, 1, 1]), /each picture needs at least one/u);
+ const bound = bindStillVideoSource(
+ { frameRate: { numerator: 30, denominator: 1 }, frameCount: 2, output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: [{ startFrame: 0, endFrameExclusive: 1 }, { startFrame: 1, endFrameExclusive: 2 }] },
+ { kind: "blob", resource: fixtureResource("still:first"), size: 1, mediaType: "image/png" },
+ );
+ assert.equal(bound.segments[0]?.source?.mediaType, "image/png");
+ assert.equal(bound.segments[1]?.source, undefined, "pictures bind in order, one per step");
+});
diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.ts
index bbf87a17..3c8c0c35 100644
--- a/packages/provider-media-local/test/provider.test.ts
+++ b/packages/provider-media-local/test/provider.test.ts
@@ -620,11 +620,11 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
mediaPipelineCapabilities.renderStill,
artifactTypes.blob,
canonicalize({
- source: extractedFrame,
request: {
frameRate: { numerator: 30, denominator: 1 },
frameCount: 15,
output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: [{ startFrame: 0, endFrameExclusive: 15, source: extractedFrame }],
},
}),
));
@@ -633,6 +633,26 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame
assert.equal(stillInspection.streams.length, 1);
assert.equal(stillInspection.streams[0]?.kind, "video");
assert.equal(stillInspection.streams[0]?.decodedUnitCount, 15);
+
+ const spread = await executeArtifact(need(
+ "need:render-still-spread",
+ mediaPipelineCapabilities.renderStill,
+ artifactTypes.blob,
+ canonicalize({
+ request: {
+ frameRate: { numerator: 30, denominator: 1 },
+ frameCount: 15,
+ output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },
+ segments: [
+ { startFrame: 0, endFrameExclusive: 10, source: extractedFrame },
+ { startFrame: 10, endFrameExclusive: 15, source: extractedFrame },
+ ],
+ },
+ }),
+ ));
+ const spreadInspection = await inspectArtifact(resources, spread);
+ assert.equal(spreadInspection.streams.length, 1);
+ assert.equal(spreadInspection.streams[0]?.decodedUnitCount, 15, "two pictures spread over the same frame count");
const stillSelection = selectMediaStreams(stillInspection, sealMediaSelectionRequest({
video: { mode: "primary-moving" },
audio: { mode: "none" },