Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(media-pipeline): StillVideo spreads several pictures over one literal duration
kashorind6ab873bb111
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.tsindex 47d2c97f..a51fd08a 100644--- a/packages/media-execution/src/execute.ts+++ b/packages/media-execution/src/execute.ts@@ -483,9 +483,9 @@ function extractFrameNeed(value: CanonicalValue): ExtractFrameNeed {function renderStillVideoNeed(value: CanonicalValue): RenderStillVideoNeed {const item = object(value, "RenderStillVideoNeed") as unknown as RenderStillVideoNeed;- assert(item.source?.kind === "blob" && item.source.mediaType.startsWith("image/"),- "RenderStillVideoNeed source must be an image Artifact");verifyStillVideoRequest(item.request);+ assert(item.request.segments.every((segment) => segment.source?.kind === "blob" && segment.source.mediaType.startsWith("image/")),+ "RenderStillVideoNeed needs an image Artifact for every segment");return item;}@@ -729,7 +729,11 @@ export async function executeNormalizeMedia(}}-/** Encode one authored image into an exact silent CFR video. */+/**+ * Encode authored images into an exact silent CFR video. One picture is held for the whole frame+ * count; several are each held for their planned segment, fitted into the first picture's frame+ * and letterboxed on black, then concatenated in order.+ */export async function executeRenderStillVideo(env: MediaExecutionEnvironment,constraints: CanonicalValue,@@ -737,22 +741,54 @@ export async function executeRenderStillVideo(const need = renderStillVideoNeed(constraints);const work = await mkdtemp(join(tmpdir(), "hypit-media-still-"));try {- const input = join(work, "source.image");const output = join(work, "still.mp4");- await stageArtifact(env, need.source, input);- const fps = `${need.request.frameRate.numerator}/${need.request.frameRate.denominator}`;- const filter = [+ const staged = new Map<string, string>();+ const inputs: string[] = [];+ for (const segment of need.request.segments) {+ const source = segment.source!;+ let path = staged.get(source.resource);+ if (path === undefined) {+ path = join(work, `picture-${staged.size}.image`);+ await stageArtifact(env, source, path);+ staged.set(source.resource, path);+ }+ inputs.push(path);+ }+ const { numerator, denominator } = need.request.frameRate;+ const fps = `${numerator}/${denominator}`;+ const hold = (frames: number): string => ["select=eq(n\\,0)","loop=loop=-1:size=1:start=0",- `trim=start_frame=0:end_frame=${need.request.frameCount}`,- `setpts=N*${need.request.frameRate.denominator}/(${need.request.frameRate.numerator}*TB)`,- "pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black",- "setsar=1",+ `trim=start_frame=0:end_frame=${frames}`,+ `setpts=N*${denominator}/(${numerator}*TB)`,].join(",");+ let argv: string[];+ if (need.request.segments.length === 1) {+ const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`;+ argv = ["-y", "-i", inputs[0]!, "-map", "0:v:0", "-an", "-vf", filter];+ } else {+ const first = await outputInspection({+ path: inputs[0]!,+ mediaType: need.request.segments[0]!.source!.mediaType,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ const picture = first.streams.find((item): item is MediaVideoStream => item.kind === "video");+ assert(picture !== undefined, "Still video first picture has no decodable image");+ const width = Math.ceil(picture.width / 2) * 2;+ const height = Math.ceil(picture.height / 2) * 2;+ const chains = need.request.segments.map((segment, index) =>+ `[${index}:v]${hold(segment.endFrameExclusive - segment.startFrame)},`+ + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p[s${index}]`);+ const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[v]`;+ argv = ["-y", ...inputs.flatMap((path) => ["-i", path]), "-filter_complex", `${chains.join(";")};${concat}`, "-map", "[v]", "-an"];+ }await runProcess({executable: env.ffmpegPath,argv: [- "-y", "-i", input, "-map", "0:v:0", "-an", "-vf", filter,+ ...argv,"-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr","-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p","-movflags", "+faststart", output,diff --git a/packages/media-pipeline/README.md b/packages/media-pipeline/README.mdindex fba8f51a..5e91acad 100644--- a/packages/media-pipeline/README.md+++ b/packages/media-pipeline/README.md@@ -52,10 +52,24 @@ A still image becomes ordinary video before it enters that waist:video="primary-moving" audio="none" span-authority="video" clock={clock}/>```-`StillVideo` returns a silent MP4 `BlobArtifact`, not `SynchronizedMedia`. Encoding its first decoded-image frame is a `render-still-video` Need; inspection and normalization remain the same explicit-steps used by imported or generated moving video. The Blob can therefore serve B-roll through-Normalize, or continue into either estimated or measured A-roll semantics afterward.+Several images spread over one literal duration the same way, each held for its share of the frames:++```svml+<media:StillVideo id="kitchen-stills" duration="6" clock={clock}>+ <media:Still source={counter.image}/>+ <media:Still source={basil.image} weight="2"/>+ <media:Still source={board.image}/>+</media:StillVideo>+```++`StillVideo` returns a silent MP4 `BlobArtifact`, not `SynchronizedMedia`. The Surface publishes the+duration and the weights as Records; `plan-still-video` divides the whole frame count among the+pictures (every picture holds at least one frame, the rest go by weight with leftovers to the largest+remainders, earlier first), `bind-still-video-source` attaches each picture in authored order, and+encoding is one `render-still-video` Need. Pictures of different sizes are fitted into the first one's+frame and letterboxed. Inspection and normalization remain the same explicit steps used by imported+or generated moving video, so the Blob can serve B-roll through Normalize, or continue into either+estimated or measured A-roll semantics afterward.Four ordinary author operations reuse that same inspection/execution boundary:diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.tsindex 2d8bab94..48c14038 100644--- a/packages/media-pipeline/src/component.ts+++ b/packages/media-pipeline/src/component.ts@@ -22,7 +22,10 @@ import {import {selectAudioStream,selectVideoStream,+ bindStillVideoSource,+ planStillVideoSegments,sealStillVideoRequest,+ verifyStillVideoLayout,verifyAudioExtractionRequest,verifyFrameExtractionRequest,verifyMediaTransformProgram,@@ -40,6 +43,7 @@ import type {ProjectSpeechEvidenceAudioNeed,RenderAudioNeed,RenderStillVideoNeed,+ StillVideoLayout,StillVideoRequest,TransformMediaNeed,} from "./types.js";@@ -86,6 +90,12 @@ export const mediaPipelineComponent = {verifyFrameExtractionRequest(inline(value, "FrameExtractionRequest"));},},+ {+ type: mediaPipelineTypes.stillVideoLayout,+ handler: ({ value }) => {+ verifyStillVideoLayout(inline(value, "StillVideoLayout"));+ },+ },{type: mediaPipelineTypes.stillVideoRequest,handler: ({ value }) => {@@ -198,6 +208,8 @@ export const mediaPipelineComponent = {const clock = inline(inputs.clock!.value, "ProgramClock") as unknown as ProgramClock;assertSpeechDurationIdentity(duration);assertProgramClockIdentity(clock);+ const layout = inline(inputs.layout!.value, "StillVideoLayout") as unknown as StillVideoLayout;+ verifyStillVideoLayout(layout);const frames = Math.round(duration * clock.frameRate.numerator / clock.frameRate.denominator);if (!Number.isSafeInteger(frames) || frames < 1) {throw new Error("Still video duration does not produce a positive safe frame count");@@ -206,18 +218,29 @@ export const mediaPipelineComponent = {frameRate: clock.frameRate,frameCount: frames,output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },+ segments: planStillVideoSegments(frames, layout.weights),});return { outputs: { request: { kind: "inline", value: canonicalize(request) } }, needs: {} };},},{- producer: mediaPipelineProducers.renderStill,+ producer: mediaPipelineProducers.bindStill,handler: ({ inputs }) => {+ const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;const source = blob(inputs.source!.value, "Still video source");+ const bound = bindStillVideoSource(request, source);+ return { outputs: { request: { kind: "inline", value: canonicalize(bound) } }, needs: {} };+ },+ },+ {+ producer: mediaPipelineProducers.renderStill,+ handler: ({ inputs }) => {const request = inline(inputs.request!.value, "StillVideoRequest") as unknown as StillVideoRequest;- if (!source.mediaType.startsWith("image/")) throw new Error("Still video source must be an image Artifact");verifyStillVideoRequest(request);- const need: RenderStillVideoNeed = { source, request };+ if (request.segments.some((segment) => segment.source === undefined)) {+ throw new Error("Still video has a segment without a picture");+ }+ const need: RenderStillVideoNeed = { request };return { outputs: {}, needs: { video: canonicalize(need) } };},},diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.tsindex c4a927b1..18651440 100644--- a/packages/media-pipeline/src/fragment.ts+++ b/packages/media-pipeline/src/fragment.ts@@ -121,29 +121,47 @@ export const extractFrameFragment = sealGraphFragment({}],});-export const stillVideoFragment = sealGraphFragment({- inputs: [- { name: "source", type: artifactTypes.blob },- { name: "duration", type: speechTypes.duration },- { name: "clock", type: programSpaceTypes.clock },- ],- operations: [- {- id: "plan",- producer: mediaPipelineProducers.planStill,- inputs: { duration: input("duration"), clock: input("clock") },- result: { kind: "output", name: "request" },- },- {- id: "render",- producer: mediaPipelineProducers.renderStill,- inputs: { source: input("source"), request: operation("plan") },- result: { kind: "need", name: "video" },- },- ],- exports: [{- name: "video",- type: artifactTypes.blob,- root: operation("render"),- }],-});+/**+ * A still video over `count` pictures: plan the frame split, bind each picture to its segment in+ * authored order, render once. The shape depends only on the count, so two elements with the same+ * number of pictures share one Fragment identity.+ */+export function createStillVideoFragment(count: number) {+ if (!Number.isSafeInteger(count) || count < 1) throw new Error("A still video needs at least one picture");+ const sources = Array.from({ length: count }, (_, index) => `source-${index}`);+ return sealGraphFragment({+ inputs: [+ { name: "duration", type: speechTypes.duration },+ { name: "clock", type: programSpaceTypes.clock },+ { name: "layout", type: mediaPipelineTypes.stillVideoLayout },+ ...sources.map((name) => ({ name, type: artifactTypes.blob })),+ ],+ operations: [+ {+ id: "plan",+ producer: mediaPipelineProducers.planStill,+ inputs: { duration: input("duration"), clock: input("clock"), layout: input("layout") },+ result: { kind: "output", name: "request" },+ },+ ...sources.map((name, index) => ({+ id: `bind-${index}`,+ producer: mediaPipelineProducers.bindStill,+ inputs: { request: operation(index === 0 ? "plan" : `bind-${index - 1}`), source: input(name) },+ result: { kind: "output" as const, name: "request" },+ })),+ {+ id: "render",+ producer: mediaPipelineProducers.renderStill,+ inputs: { request: operation(`bind-${count - 1}`) },+ result: { kind: "need", name: "video" },+ },+ ],+ exports: [{+ name: "video",+ type: artifactTypes.blob,+ root: operation("render"),+ }],+ });+}++export const stillVideoFragment = createStillVideoFragment(1);diff --git a/packages/media-pipeline/src/index.ts b/packages/media-pipeline/src/index.tsindex bc1d13ba..5439410d 100644--- a/packages/media-pipeline/src/index.ts+++ b/packages/media-pipeline/src/index.ts@@ -7,6 +7,7 @@ export {extractFrameFragment,synchronizedMediaFragment,stillVideoFragment,+ createStillVideoFragment,transformMediaFragment,} from "./fragment.js";export {@@ -30,12 +31,16 @@ export {sealAudioExtractionRequest,sealFrameExtractionRequest,sealMediaTransformProgram,+ sealStillVideoLayout,sealStillVideoRequest,+ planStillVideoSegments,+ bindStillVideoSource,selectAudioStream,selectVideoStream,verifyAudioExtractionRequest,verifyFrameExtractionRequest,verifyMediaTransformProgram,+ verifyStillVideoLayout,verifyStillVideoRequest,} from "./operations.js";export type * from "./types.js";diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.tsindex 1037d07c..88ca2df3 100644--- a/packages/media-pipeline/src/manifest.ts+++ b/packages/media-pipeline/src/manifest.ts@@ -21,6 +21,7 @@ export const mediaPipelineTypes = {audioExtractionRequest: { module: mediaPipelineModuleRef, name: "AudioExtractionRequest" },frameExtractionRequest: { module: mediaPipelineModuleRef, name: "FrameExtractionRequest" },stillVideoRequest: { module: mediaPipelineModuleRef, name: "StillVideoRequest" },+ stillVideoLayout: { module: mediaPipelineModuleRef, name: "StillVideoLayout" },} satisfies Record<string, TypeRef>;export const mediaPipelineCapabilities = {inspect: { module: mediaPipelineModuleRef, name: "inspect-media" },@@ -42,6 +43,7 @@ export const mediaPipelineProducers = {extractFrame: { module: mediaPipelineModuleRef, name: "request-frame-extraction" },planStill: { module: mediaPipelineModuleRef, name: "plan-still-video" },renderStill: { module: mediaPipelineModuleRef, name: "request-still-video" },+ bindStill: { module: mediaPipelineModuleRef, name: "bind-still-video-source" },projectSpeechEvidenceAudio: { module: mediaPipelineModuleRef, name: "request-speech-evidence-audio" },planAudio: { module: mediaPipelineModuleRef, name: "compile-audio-program" },renderAudio: { module: mediaPipelineModuleRef, name: "request-audio-render" },@@ -168,6 +170,29 @@ export const stillVideoRequestSchema: ValueSchema = {pixelFormat: { schema: { kind: "literal", value: "yuv420p" } },},} },+ segments: { schema: { kind: "array", items: {+ kind: "object",+ fields: {+ startFrame: { schema: { kind: "number", integer: true, minimum: 0 } },+ endFrameExclusive: { schema: { kind: "number", integer: true, minimum: 1 } },+ source: { optional: true, schema: {+ kind: "object",+ fields: {+ kind: { schema: { kind: "literal", value: "blob" } },+ resource: { schema: { kind: "string", minLength: 5, maxLength: 256 } },+ size: { schema: { kind: "number", integer: true, minimum: 0 } },+ mediaType: { schema: { kind: "string", minLength: 6 } },+ },+ } },+ },+ } } },+ },+};++export const stillVideoLayoutSchema: ValueSchema = {+ kind: "object",+ fields: {+ weights: { schema: { kind: "array", items: { kind: "number", minimum: 0 } } },},};@@ -265,24 +290,39 @@ export const mediaPipelineMarkupSurfaces = [},{name: "still-video", tag: "StillVideo", mode: "structured",- outputs: [speechTypes.duration, mediaPipelineTypes.stillVideoRequest, artifactTypes.blob],+ outputs: [speechTypes.duration, mediaPipelineTypes.stillVideoLayout, mediaPipelineTypes.stillVideoRequest, artifactTypes.blob],vocabulary: {- summary: "Encodes one authored image as an ordinary silent MP4 Blob on an explicit duration and frame clock.",+ summary: "Spreads one or more authored images over an explicit duration as an ordinary silent MP4 Blob on a frame clock.",attributes: [{ name: "id", kind: "identifier", required: true,summary: "Names the still-video operation and the MP4 Artifact it publishes." },- { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],- summary: "Selects the authored image whose first decoded frame is held for the full video." },+ { name: "source", kind: "reference", required: false, accepts: [artifactTypes.blob],+ summary: "Selects the one authored image held for the full video; write Still children instead for several images." },{ name: "duration", kind: "literal", required: true,summary: "Sets the video's length in seconds, such as 6 or 2.5s; the author's decision, measured beforehand with hypit measure." },{ name: "clock", kind: "reference", required: true, accepts: [programSpaceTypes.clock],summary: "Selects the frame clock used by the generated MP4." },],+ children: [+ { tag: "Still", cardinality: "many",+ summary: "One image in authored order; the duration is divided among the Still children by weight, equally unless weights say otherwise.",+ attributes: [+ { name: "source", kind: "reference", required: true, accepts: [artifactTypes.blob],+ summary: "Selects the authored image whose first decoded frame is held for this share." },+ { name: "weight", kind: "literal", required: false,+ summary: "Sets this image's share of the duration relative to its siblings, such as 2; every Still weighs 1 unless written." },+ ] },+ ],ports: [{ name: "video", type: artifactTypes.blob,summary: "The ordinary silent MP4 Artifact, addressed as `<id>.video`." }],- example: `<media:StillVideo id="opening-still" source={opening-head}- duration={opening-duration.duration} clock={clock}/>`,+ example: `<media:StillVideo id="kitchen-stills" duration="6" clock={clock}>+ <media:Still source={counter}/>+ <media:Still source={basil} weight="2"/>+ <media:Still source={board}/>+</media:StillVideo>`,notes: [+ "Write exactly one of source or Still children. Frames are whole: each image gets the floor of its share and the leftover frames go to the largest remainders, so the split is deterministic and every image holds at least one frame.",+ "Images of different sizes are fitted into the first image's frame, letterboxed on black.","The result is a normal video Blob, not SynchronizedMedia and not a SemanticTake.","Use Normalize afterward exactly as for generated or imported moving video.","Encoding is a render-still-video Need fulfilled by the selected media Provider; this Surface never invokes FFmpeg itself.",@@ -425,6 +465,9 @@ export const mediaPipelineManifest: ModuleManifest = {{name: mediaPipelineTypes.stillVideoRequest.name,},+ {+ name: mediaPipelineTypes.stillVideoLayout.name,+ },],capabilities: [{ name: mediaPipelineCapabilities.inspect.name, returns: mediaTypes.inspection },@@ -521,14 +564,23 @@ export const mediaPipelineManifest: ModuleManifest = {inputs: [{ name: "duration", type: speechTypes.duration },{ name: "clock", type: programSpaceTypes.clock },+ { name: "layout", type: mediaPipelineTypes.stillVideoLayout },],outputs: [{ name: "request", type: mediaPipelineTypes.stillVideoRequest }],needs: [],},{- name: mediaPipelineProducers.renderStill.name,+ name: mediaPipelineProducers.bindStill.name,inputs: [+ { name: "request", type: mediaPipelineTypes.stillVideoRequest },{ name: "source", type: artifactTypes.blob },+ ],+ outputs: [{ name: "request", type: mediaPipelineTypes.stillVideoRequest }],+ needs: [],+ },+ {+ name: mediaPipelineProducers.renderStill.name,+ inputs: [{ name: "request", type: mediaPipelineTypes.stillVideoRequest },],outputs: [],diff --git a/packages/media-pipeline/src/operations.ts b/packages/media-pipeline/src/operations.tsindex ae8d6554..ad14e19c 100644--- a/packages/media-pipeline/src/operations.ts+++ b/packages/media-pipeline/src/operations.ts@@ -1,6 +1,7 @@import { verifyMediaInspection } from "@hypit/media";import type { MediaAudioStream, MediaInspection, MediaVideoStream } from "@hypit/media";import { canonicalize } from "@hypit/protocol";+import type { BlobRef } from "@hypit/protocol";import type {AudioExtractionRequest,@@ -8,7 +9,9 @@ import type {MediaAudioSelector,MediaTransformProgram,MediaVideoSelector,+ StillVideoLayout,StillVideoRequest,+ StillVideoSegment,} from "./types.js";function assert(condition: unknown, message: string): asserts condition {@@ -114,6 +117,51 @@ export function sealFrameExtractionRequest(value: FrameExtractionRequest): Framereturn result;}+export function verifyStillVideoLayout(value: unknown): asserts value is StillVideoLayout {+ const layout = object(value, "StillVideoLayout");+ assert(Array.isArray(layout.weights) && layout.weights.length > 0, "StillVideoLayout needs at least one weight");+ for (const weight of layout.weights) {+ assert(typeof weight === "number" && Number.isFinite(weight) && weight > 0, "StillVideoLayout weights must be positive");+ }+}++export function sealStillVideoLayout(value: StillVideoLayout): StillVideoLayout {+ const result = canonicalize(value) as unknown as StillVideoLayout;+ verifyStillVideoLayout(result);+ return result;+}++/**+ * Spread `frameCount` whole frames over the weights. Every picture first holds one frame; the rest+ * are shared by weight, each picture taking the floor of its share and the frames left over going+ * one each to the largest fractional remainders, earlier picture first on a tie. Deterministic, and+ * refused when there are fewer frames than pictures.+ */+export function planStillVideoSegments(frameCount: number, weights: readonly number[]): readonly StillVideoSegment[] {+ assert(Number.isSafeInteger(frameCount) && frameCount > 0, "Still video frame count must be positive");+ assert(weights.length > 0, "Still video needs at least one picture");+ assert(frameCount >= weights.length, `Still video has ${frameCount} frames for ${weights.length} pictures; each picture needs at least one`);+ const total = weights.reduce((sum, weight) => sum + weight, 0);+ const shared = frameCount - weights.length;+ const exact = weights.map((weight) => shared * weight / total);+ const floors = exact.map((share) => 1 + Math.floor(share));+ let remaining = frameCount - floors.reduce((sum, item) => sum + item, 0);+ const order = exact.map((share, index) => ({ index, fraction: share - Math.floor(share) }))+ .sort((left, right) => right.fraction - left.fraction || left.index - right.index);+ for (const { index } of order) {+ if (remaining === 0) break;+ floors[index] = floors[index]! + 1;+ remaining -= 1;+ }+ const segments: StillVideoSegment[] = [];+ let start = 0;+ for (const frames of floors) {+ segments.push({ startFrame: start, endFrameExclusive: start + frames });+ start += frames;+ }+ return segments;+}+export function verifyStillVideoRequest(value: unknown): asserts value is StillVideoRequest {const request = object(value, "StillVideoRequest");const frameRate = object(request.frameRate, "StillVideoRequest.frameRate");@@ -125,6 +173,21 @@ export function verifyStillVideoRequest(value: unknown): asserts value is StillVconst output = object(request.output, "StillVideoRequest.output");assert(output.container === "mp4" && output.codec === "h264" && output.pixelFormat === "yuv420p","StillVideoRequest output profile is unsupported");+ assert(Array.isArray(request.segments) && request.segments.length > 0, "StillVideoRequest needs at least one segment");+ let cursor = 0;+ for (const item of request.segments) {+ const segment = object(item, "StillVideoRequest.segments[]");+ assert(segment.startFrame === cursor, "StillVideoRequest segments must be contiguous from frame 0");+ assert(Number.isSafeInteger(segment.endFrameExclusive) && (segment.endFrameExclusive as number) > cursor,+ "StillVideoRequest segments must each hold at least one frame");+ cursor = segment.endFrameExclusive as number;+ if (segment.source !== undefined) {+ const source = object(segment.source, "StillVideoRequest.segments[].source");+ assert(source.kind === "blob" && typeof source.mediaType === "string" && source.mediaType.startsWith("image/"),+ "StillVideoRequest segment source must be an image Artifact");+ }+ }+ assert(cursor === request.frameCount, "StillVideoRequest segments must cover exactly frameCount frames");}export function sealStillVideoRequest(value: StillVideoRequest): StillVideoRequest {@@ -133,6 +196,18 @@ export function sealStillVideoRequest(value: StillVideoRequest): StillVideoRequereturn result;}+/** Attach one picture to the first segment that has none; the pictures arrive in authored order. */+export function bindStillVideoSource(request: StillVideoRequest, source: BlobRef): StillVideoRequest {+ verifyStillVideoRequest(request);+ assert(source.kind === "blob" && source.mediaType.startsWith("image/"), "Still video source must be an image Artifact");+ const index = request.segments.findIndex((segment) => segment.source === undefined);+ assert(index >= 0, "Still video already has a picture for every segment");+ return sealStillVideoRequest({+ ...request,+ segments: request.segments.map((segment, at) => at === index ? { ...segment, source } : segment),+ });+}+function uniqueDefault<T extends { readonly disposition: { readonly default: boolean }; readonly index: number }>(values: readonly T[],subject: string,diff --git a/packages/media-pipeline/src/surface.ts b/packages/media-pipeline/src/surface.tsindex e3b0859b..c1e94e06 100644--- a/packages/media-pipeline/src/surface.ts+++ b/packages/media-pipeline/src/surface.ts@@ -17,7 +17,7 @@ import { svsRecipeType, type SvsRecipe } from "@hypit/svs";import {extractAudioFragment,extractFrameFragment,- stillVideoFragment,+ createStillVideoFragment,synchronizedMediaFragment,transformMediaFragment,} from "./fragment.js";@@ -228,36 +228,76 @@ export const decodeSynchronizedMediaSurface: StructuredSurfaceHandler = ({ eleme};};+/** The pictures of a StillVideo in authored order, with their weights: one `source`, or Still children. */+function stillPictures(+ element: StructuredElement,+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,+): readonly { readonly source: SurfaceResolvedReference; readonly weight: number }[] {+ const children = element.children.filter((child) => child.kind === "element" || child.value.trim().length > 0);+ if (element.attributes.source !== undefined) {+ if (children.length > 0) throw new Error(`${element.name} takes either source or Still children, not both.`);+ return [{ source: ref(element.attributes.source, `${element.name}.source`, resolveReference), weight: 1 }];+ }+ const pictures = children.map((child) => {+ if (child.kind !== "element" || localName(child.name) !== "Still") {+ throw new Error(`${element.name} accepts only Still children.`);+ }+ exactAttributes(child, ["source"], ["weight"]);+ empty(child);+ const rawWeight = child.attributes.weight;+ let weight = 1;+ if (rawWeight !== undefined) {+ if (typeof rawWeight !== "string") throw new Error(`${child.name}.weight must be text.`);+ weight = Number(rawWeight);+ if (!Number.isFinite(weight) || weight <= 0) throw new Error(`${child.name}.weight must be a positive number.`);+ }+ return { source: ref(child.attributes.source, `${child.name}.source`, resolveReference), weight };+ });+ if (pictures.length === 0) throw new Error(`${element.name} requires a source or at least one Still child.`);+ return pictures;+}+export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {- exactAttributes(element, ["id", "source", "duration", "clock"]);- empty(element);+ exactAttributes(element, ["id", "duration", "clock"], ["source"]);const id = text(element, "id");- const source = ref(element.attributes.source, `${element.name}.source`, resolveReference);+ const pictures = stillPictures(element, resolveReference);// The duration is the author's decision, written here; it is published as an ordinary// SpeechDuration Record so the fragment's plan step reads the same value a graph edge would carry.const durationSec = seconds(text(element, "duration"), `${element.name}.duration`, false);const durationId = `${id}.duration`;+ const layoutId = `${id}.layout`;const clock = typedReference(element.attributes.clock,`${element.name}.clock`,programSpaceTypes.clock,resolveReference,);+ const fragment = createStillVideoFragment(pictures.length);return {records: [{id: durationId,type: speechTypes.duration,value: { kind: "inline", value: durationSec },range: element.range,+ }, {+ id: layoutId,+ type: mediaPipelineTypes.stillVideoLayout,+ value: { kind: "inline", value: { weights: pictures.map((picture) => picture.weight) } },+ range: element.range,}],components: [{id,- fragment: stillVideoFragment.id,- inputs: { source: source.ref, duration: { kind: "record", id: durationId }, clock: clock.ref },+ fragment: fragment.id,+ inputs: {+ duration: { kind: "record", id: durationId },+ clock: clock.ref,+ layout: { kind: "record", id: layoutId },+ ...Object.fromEntries(pictures.map((picture, index) => [`source-${index}`, picture.source.ref])),+ },outputs: { video: `${id}.video` },range: element.range,}],- fragments: [stillVideoFragment],+ fragments: [fragment],};};diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.tsindex 6773b4e6..4744025a 100644--- a/packages/media-pipeline/src/types.ts+++ b/packages/media-pipeline/src/types.ts@@ -67,7 +67,19 @@ export type FrameExtractionRequest = {readonly output: { readonly format: "png" };};-/** Exact silent video requested from one authored still image. */+/** How the still video divides its literal duration among its pictures: one weight per picture, in order. */+export type StillVideoLayout = {+ readonly weights: readonly number[];+};++/** One picture's span of the still video, in frames; `source` is bound one picture at a time. */+export type StillVideoSegment = {+ readonly startFrame: number;+ readonly endFrameExclusive: number;+ readonly source?: BlobRef;+};++/** Exact silent video requested from one or more authored still images spread over a literal duration. */export type StillVideoRequest = {readonly frameRate: MediaRational;readonly frameCount: number;@@ -76,6 +88,7 @@ export type StillVideoRequest = {readonly codec: "h264";readonly pixelFormat: "yuv420p";};+ readonly segments: readonly StillVideoSegment[];};export type TransformMediaNeed = {@@ -97,8 +110,8 @@ export type ExtractFrameNeed = {readonly output: FrameExtractionRequest["output"];};+/** Every segment of the request carries its picture by the time the Need is made. */export type RenderStillVideoNeed = {- readonly source: BlobRef;readonly request: StillVideoRequest;};diff --git a/packages/media-pipeline/test/manifest.test.ts b/packages/media-pipeline/test/manifest.test.tsindex 102b7a0b..c1467558 100644--- a/packages/media-pipeline/test/manifest.test.ts+++ b/packages/media-pipeline/test/manifest.test.ts@@ -5,6 +5,7 @@ import { speechTypes } from "@hypit/speech";import { svsRecipeType } from "@hypit/svs";import assert from "node:assert/strict";import test from "node:test";+import { fixtureResource } from "../../../test/fixture-resource.js";import {mediaPipelineTypes,@@ -13,6 +14,9 @@ import {decodeExtractFrameSurface,decodeTransformMediaSurface,decodeStillVideoSurface,+ createStillVideoFragment,+ planStillVideoSegments,+ bindStillVideoSource,extractAudioFragment,extractFrameFragment,synchronizedMediaFragment,@@ -145,5 +149,64 @@ test("StillVideo stops at an ordinary MP4 branch before Normalize", async () =>id: "opening-still.duration", type: speechTypes.duration, value: { kind: "inline", value: 6 }, range,}, "the literal duration is the author's, published as an ordinary SpeechDuration Record");assert.deepEqual(output.components[0]?.inputs.duration, { kind: "record", id: "opening-still.duration" });+ assert.deepEqual(output.components[0]?.inputs["source-0"], { kind: "record", id: "opening-head" });+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1] } }, "one picture, one whole share");assert.equal(stillVideoFragment.exports[0]?.type.name, artifactTypes.blob.name);});++test("StillVideo spreads several pictures over one literal duration by weight, in authored order", async () => {+ const range = { source: "still.svml", start: 0, end: 1 };+ const resolveReference = (path: string) => path === "clock"+ ? { path, ref: { kind: "record" as const, id: path }, type: programSpaceTypes.clock }+ : { path, ref: { kind: "record" as const, id: path }, type: artifactTypes.blob };+ const child = (source: string, weight?: string) => ({+ kind: "element" as const, name: "media:Still",+ attributes: { source: { kind: "reference" as const, path: source }, ...(weight === undefined ? {} : { weight }) },+ children: [], range,+ });+ const output = await decodeStillVideoSurface({+ sourceName: range.source,+ element: {+ kind: "element", name: "media:StillVideo",+ attributes: { id: "kitchen", duration: "6", clock: { kind: "reference", path: "clock" } },+ children: [child("counter"), child("basil", "2"), child("board")],+ range,+ },+ resolveReference,+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },+ });+ assert.equal(output.fragments[0]?.id, createStillVideoFragment(3).id);+ assert.notEqual(output.fragments[0]?.id, stillVideoFragment.id, "three pictures are a different shape from one");+ assert.deepEqual(output.records[1]?.value, { kind: "inline", value: { weights: [1, 2, 1] } });+ assert.deepEqual(+ ["source-0", "source-1", "source-2"].map((name) => output.components[0]?.inputs[name]),+ [{ kind: "record", id: "counter" }, { kind: "record", id: "basil" }, { kind: "record", id: "board" }],+ );+ await assert.rejects(async () => await decodeStillVideoSurface({+ sourceName: range.source,+ element: {+ kind: "element", name: "media:StillVideo",+ attributes: { id: "both", source: { kind: "reference", path: "counter" }, duration: "6", clock: { kind: "reference", path: "clock" } },+ children: [child("basil")],+ range,+ },+ resolveReference,+ resolveAsset: async () => { throw new Error("no asset resolution expected"); },+ }), /either source or Still children/u);+});++test("still video frames are whole and every picture holds at least one", () => {+ assert.deepEqual(planStillVideoSegments(180, [1, 2, 1]).map((item) => item.endFrameExclusive - item.startFrame), [45, 90, 45]);+ assert.deepEqual(planStillVideoSegments(10, [1, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [4, 3, 3],+ "the leftover frame goes to the earliest of equal remainders");+ assert.deepEqual(planStillVideoSegments(10, [1, 1.5, 1]).map((item) => item.endFrameExclusive - item.startFrame), [3, 4, 3]);+ assert.deepEqual(planStillVideoSegments(3, [100, 1, 1]).map((item) => item.endFrameExclusive - item.startFrame), [1, 1, 1]);+ assert.throws(() => planStillVideoSegments(2, [1, 1, 1]), /each picture needs at least one/u);+ const bound = bindStillVideoSource(+ { frameRate: { numerator: 30, denominator: 1 }, frameCount: 2, output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },+ segments: [{ startFrame: 0, endFrameExclusive: 1 }, { startFrame: 1, endFrameExclusive: 2 }] },+ { kind: "blob", resource: fixtureResource("still:first"), size: 1, mediaType: "image/png" },+ );+ assert.equal(bound.segments[0]?.source?.mediaType, "image/png");+ assert.equal(bound.segments[1]?.source, undefined, "pictures bind in order, one per step");+});diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.tsindex bbf87a17..3c8c0c35 100644--- a/packages/provider-media-local/test/provider.test.ts+++ b/packages/provider-media-local/test/provider.test.ts@@ -620,11 +620,11 @@ test("local media Provider transforms A/V and extracts ordinary audio and framemediaPipelineCapabilities.renderStill,artifactTypes.blob,canonicalize({- source: extractedFrame,request: {frameRate: { numerator: 30, denominator: 1 },frameCount: 15,output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },+ segments: [{ startFrame: 0, endFrameExclusive: 15, source: extractedFrame }],},}),));@@ -633,6 +633,26 @@ test("local media Provider transforms A/V and extracts ordinary audio and frameassert.equal(stillInspection.streams.length, 1);assert.equal(stillInspection.streams[0]?.kind, "video");assert.equal(stillInspection.streams[0]?.decodedUnitCount, 15);++ const spread = await executeArtifact(need(+ "need:render-still-spread",+ mediaPipelineCapabilities.renderStill,+ artifactTypes.blob,+ canonicalize({+ request: {+ frameRate: { numerator: 30, denominator: 1 },+ frameCount: 15,+ output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },+ segments: [+ { startFrame: 0, endFrameExclusive: 10, source: extractedFrame },+ { startFrame: 10, endFrameExclusive: 15, source: extractedFrame },+ ],+ },+ }),+ ));+ const spreadInspection = await inspectArtifact(resources, spread);+ assert.equal(spreadInspection.streams.length, 1);+ assert.equal(spreadInspection.streams[0]?.decodedUnitCount, 15, "two pictures spread over the same frame count");const stillSelection = selectMediaStreams(stillInspection, sealMediaSelectionRequest({video: { mode: "primary-moving" },audio: { mode: "none" },