Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
refactor(stand-in): compose cards through still video
kashorine43ad670858d
diff --git a/packages/media-execution/src/card.ts b/packages/media-execution/src/card.tsindex d3fb067e..a9eb8ab3 100644--- a/packages/media-execution/src/card.ts+++ b/packages/media-execution/src/card.ts@@ -1,9 +1,5 @@-import { mkdtemp, rm, writeFile } from "node:fs/promises";-import { tmpdir } from "node:os";-import { join } from "node:path";import { deflateSync } from "node:zlib";-import type { MediaStream } from "@hypit/media";import type { CanonicalValue } from "@hypit/protocol";import { verifyStandInCardRequest } from "@hypit/stand-in";import type { StandInCardRequest } from "@hypit/stand-in";@@ -11,19 +7,13 @@ import type { StandInCardRequest } from "@hypit/stand-in";/*** A generic stand-in card, drawn from only the media shape a Run supplies.*- * The card is pixels, not a model call: a flat ground coloured by kind, a thick border, a diagonal- * STAND-IN watermark, the kind in large type, the frame and duration, and for video a running- * timecode and progress bar. Everything is drawn here with a- * built-in 5x7 bitmap font so the same card comes out of every machine, with or without a font- * library in its ffmpeg. The PNG is deterministic for a given request.+ * The card is pixels, not a model call. Everything is drawn here with a built-in 5x7 bitmap font,+ * so the same card comes out of every machine. The PNG is deterministic for a given request.*/type Rgb = readonly [number, number, number];-const GROUND: Record<StandInCardRequest["kind"], Rgb> = {- video: [0x1E, 0x3A, 0x5F],- image: [0x2E, 0x4A, 0x3A],-};+const GROUND: Rgb = [0x17, 0x23, 0x32];const ACCENT: Rgb = [0xF5, 0xC5, 0x42];const TEXT: Rgb = [0xFF, 0xFF, 0xFF];const MUTED: Rgb = [0xC9, 0xD1, 0xD9];@@ -153,23 +143,6 @@ class Raster {return cursor - x;}- /** The same text stepped along a 45° line, one glyph per step; the watermark reads diagonally. */- diagonalText(value: string, x: number, y: number, scale: number, color: Rgb, alpha: number): void {- const step = GLYPH_WIDTH * scale;- let index = 0;- for (const character of value) {- const rows = glyph(character);- const originX = x + index * step;- const originY = y + index * step;- for (let row = 0; row < GLYPH_HEIGHT; row += 1) {- for (let column = 0; column < GLYPH_WIDTH; column += 1) {- if (rows[row]![column] === "#") this.fill(originX + column * scale, originY + row * scale, scale, scale, color, alpha);- }- }- index += 1;- }- }-png(): Uint8Array {const stride = this.width * 3;const scanlines = Buffer.alloc(this.height * (1 + stride));@@ -215,11 +188,17 @@ function textWidth(value: string, scale: number): number {return Math.max(0, [...value].length * ADVANCE * scale - scale);}-function seconds(video: NonNullable<StandInCardRequest["video"]>): number {- return video.frameCount * video.frameRate.denominator / video.frameRate.numerator;-}+export type ClipTimeGuideRequest = {+ readonly width: number;+ readonly height: number;+ readonly frameRate: { readonly numerator: number; readonly denominator: number };+ readonly frameCount: number;+};-function timecode(frame: number, video: NonNullable<StandInCardRequest["video"]>): string {+function timecode(+ frame: number,+ video: Pick<ClipTimeGuideRequest, "frameRate" | "frameCount">,+): string {const fps = video.frameRate.numerator / video.frameRate.denominator;const total = frame / fps;const minutes = Math.floor(total / 60);@@ -231,33 +210,29 @@ function timecode(frame: number, video: NonNullable<StandInCardRequest["video"]>type Layout = {readonly border: number;- readonly band: number;readonly margin: number;};function layout(request: StandInCardRequest): Layout {const m = Math.min(request.width, request.height);const border = Math.max(6, Math.round(m * 0.03));- const band = request.kind === "video" ? Math.max(36, Math.round(request.height * 0.06)) : 0;- return { border, band, margin: border * 2 };+ return { border, margin: border * 2 };}-/** The still part of the card: everything but the timecode band. */+/** Draw one neutral Card. Timing, if needed, belongs to the StillVideo that displays it. */export function drawStandInCard(request: StandInCardRequest): Uint8Array {verifyStandInCardRequest(request);const { width, height } = request;const m = Math.min(width, height);- const { border, band, margin } = layout(request);- const raster = new Raster(width, height, GROUND[request.kind]);+ const { border, margin } = layout(request);+ const raster = new Raster(width, height, GROUND);- // Diagonal watermark: stepped 45° lines of STAND-IN across the whole ground, faint.- const markScale = Math.max(2, Math.floor(m / 220));- const word = "STAND-IN ";- const wordSpan = word.length * GLYPH_WIDTH * markScale;- const lineGap = Math.max(GLYPH_HEIGHT * markScale * 3, Math.round(m * 0.22));- for (let start = -height; start < width; start += lineGap) {- for (let along = 0; start + along < width + wordSpan && along < width + height; along += wordSpan) {- raster.diagonalText(word, start + along, along, markScale, TEXT, 0.09);+ // A quiet visual field gives components something visible to sit over without pretending to be+ // a generated subject. The blocks scale with the requested canvas and carry no media semantics.+ const cell = Math.max(16, Math.round(m * 0.08));+ for (let y = border; y < height - border; y += cell) {+ for (let x = border; x < width - border; x += cell) {+ if (((x / cell) + (y / cell)) % 3 < 1) raster.fill(x, y, cell, cell, TEXT, 0.018);}}@@ -267,81 +242,55 @@ export function drawStandInCard(request: StandInCardRequest): Uint8Array {raster.fill(0, 0, border, height, ACCENT);raster.fill(width - border, 0, border, height, ACCENT);- // Corner tag.+ // One label is enough; the rest of the card communicates by shape and colour.const tagScale = Math.max(2, Math.floor(m / 260));raster.text("STAND-IN", margin, margin, tagScale, ACCENT);- const kindTag = request.kind === "video" ? "NOT YET GENERATED" : "NOT YET GENERATED";- raster.text(kindTag, width - margin - textWidth(kindTag, tagScale), margin, tagScale, MUTED);- // Centre block: only the generic media shape selected by the Run.- const titleScale = Math.max(4, Math.floor(m / 70));+ const frameWidth = Math.max(Math.round(width * 0.42), margin * 5);+ const frameHeight = Math.max(Math.round(height * 0.28), margin * 4);+ const frameX = Math.round((width - frameWidth) / 2);+ const frameY = Math.round((height - frameHeight) / 2);+ const stroke = Math.max(3, Math.round(border / 2));+ raster.fill(frameX, frameY, frameWidth, stroke, MUTED, 0.75);+ raster.fill(frameX, frameY + frameHeight - stroke, frameWidth, stroke, MUTED, 0.75);+ raster.fill(frameX, frameY, stroke, frameHeight, MUTED, 0.75);+ raster.fill(frameX + frameWidth - stroke, frameY, stroke, frameHeight, MUTED, 0.75);+ raster.fill(frameX + stroke * 4, frameY + frameHeight - stroke * 6,+ Math.round(frameWidth * 0.34), stroke * 2, ACCENT, 0.9);+ raster.fill(frameX + Math.round(frameWidth * 0.58), frameY + stroke * 4,+ stroke * 4, stroke * 4, TEXT, 0.55);+const lineScale = Math.max(2, Math.floor(m / 200));- const title = request.kind.toUpperCase();- const frameLine = request.video === undefined- ? `${width} X ${height}`- : `${width} X ${height} ${seconds(request.video).toFixed(2)} S ${(request.video.frameRate.numerator / request.video.frameRate.denominator).toFixed(2)} FPS`;- const blockHeight = GLYPH_HEIGHT * titleScale + lineScale * 6- + GLYPH_HEIGHT * lineScale;- let y = Math.round((height - band - blockHeight) / 2);- const centred = (value: string, scale: number, color: Rgb): void => {- raster.text(value, Math.round((width - textWidth(value, scale)) / 2), y, scale, color);- y += GLYPH_HEIGHT * scale;- };- centred(title, titleScale, TEXT);- y += lineScale * 6;- centred(frameLine, lineScale, MUTED);+ const frameLine = `${width} X ${height}`;+ raster.text(frameLine, Math.round((width - textWidth(frameLine, lineScale)) / 2),+ frameY + frameHeight + margin, lineScale, MUTED);return raster.png();}-export function drawStandInSilence(sampleFrames: number): Uint8Array {- if (!Number.isSafeInteger(sampleFrames) || sampleFrames < 1) {- throw new Error("Stand-in silence sampleFrames must be a positive integer");- }- const dataBytes = sampleFrames * 2 * 2;- const wav = Buffer.alloc(44 + dataBytes);- wav.write("RIFF", 0);- wav.writeUInt32LE(36 + dataBytes, 4);- wav.write("WAVE", 8);- wav.write("fmt ", 12);- wav.writeUInt32LE(16, 16);- wav.writeUInt16LE(1, 20);- wav.writeUInt16LE(2, 22);- wav.writeUInt32LE(48_000, 24);- wav.writeUInt32LE(48_000 * 4, 28);- wav.writeUInt16LE(4, 32);- wav.writeUInt16LE(16, 34);- wav.write("data", 36);- wav.writeUInt32LE(dataBytes, 40);- return Uint8Array.from(wav);-}-/** The band a video card carries under its picture: timecode, frame counter and progress bar. */-export function drawStandInBand(request: StandInCardRequest, frame: number): Uint8Array {- verifyStandInCardRequest(request);- const video = request.video;- if (video === undefined) throw new Error("only a video stand-in has a timecode band");- const { width } = request;- const { border, band, margin } = layout(request);+export function drawClipTimeGuide(request: ClipTimeGuideRequest, frame: number): Uint8Array {+ const { width, height, frameRate, frameCount } = request;+ if (!Number.isSafeInteger(frame) || frame < 0 || frame >= frameCount) {+ throw new Error("Clip time guide frame lies outside its clip");+ }+ const border = Math.max(6, Math.round(Math.min(width, height) * 0.03));+ const band = Math.max(36, Math.round(height * 0.06));+ const margin = border * 2;const raster = new Raster(width, band, INK);raster.fill(0, 0, border, band, ACCENT);raster.fill(width - border, 0, border, band, ACCENT);const scale = Math.max(2, Math.floor((band - border) / (GLYPH_HEIGHT + 4)));const textY = Math.round((band - border - GLYPH_HEIGHT * scale) / 2);- raster.text(timecode(frame, video), margin, textY, scale, TEXT);- const counter = `F ${frame + 1}/${video.frameCount}`;+ raster.text(timecode(frame, { frameRate, frameCount }), margin, textY, scale, TEXT);+ const counter = `F ${frame + 1}/${frameCount}`;raster.text(counter, width - margin - textWidth(counter, scale), textY, scale, MUTED);raster.fill(0, band - border, width, border, INK);- raster.fill(0, band - border, Math.round(width * (frame + 1) / video.frameCount), border, ACCENT);+ raster.fill(0, band - border, Math.round(width * (frame + 1) / frameCount), border, ACCENT);return raster.png();}export type StandInDrawingEnvironment = {- readonly ffmpegPath: string;- readonly processTimeoutMs: number;- readonly runFfmpeg: (argv: readonly string[]) => Promise<void>;- readonly inspectStreams: (path: string) => Promise<readonly MediaStream[]>;readonly putBytes: (bytes: Uint8Array, mediaType: string) => Promise<CanonicalValue>;- readonly putFile: (path: string, mediaType: string) => Promise<CanonicalValue>;};export function standInCardNeed(value: CanonicalValue): StandInCardRequest {@@ -349,45 +298,10 @@ export function standInCardNeed(value: CanonicalValue): StandInCardRequest {return value;}-/** Draw the card; a video card is the still card with its band composited frame by frame by ffmpeg. */+/** Draw the Card image. */export async function renderStandInCard(env: StandInDrawingEnvironment,request: StandInCardRequest,): Promise<CanonicalValue> {- const still = drawStandInCard(request);- if (request.video === undefined) return await env.putBytes(still, "image/png");- const video = request.video;- const work = await mkdtemp(join(tmpdir(), "hypit-stand-in-"));- try {- const base = join(work, "card.png");- await writeFile(base, still);- for (let frame = 0; frame < video.frameCount; frame += 1) {- await writeFile(join(work, `band-${String(frame).padStart(6, "0")}.png`), drawStandInBand(request, frame));- }- const fps = `${video.frameRate.numerator}/${video.frameRate.denominator}`;- const output = join(work, "card.mp4");- const silence = video.audio === "silence";- await env.runFfmpeg([- "-y",- "-loop", "1", "-framerate", fps, "-i", base,- "-framerate", fps, "-i", join(work, "band-%06d.png"),- ...(silence ? ["-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"] : []),- "-filter_complex", "[0:v][1:v]overlay=0:main_h-overlay_h:shortest=1,format=yuv420p[v]",- "-map", "[v]", ...(silence ? ["-map", "2:a:0", "-c:a", "aac", "-b:a", "96k", "-shortest"] : []),- "-frames:v", String(video.frameCount), "-r", fps, "-fps_mode", "cfr",- "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",- "-movflags", "+faststart", output,- ]);- const streams = await env.inspectStreams(output);- const visual = streams.find((item) => item.kind === "video");- const audio = streams.filter((item) => item.kind === "audio");- if (visual === undefined || visual.kind !== "video" || visual.decodedUnitCount !== video.frameCount- || visual.width !== request.width || visual.height !== request.height- || streams.length !== 1 + audio.length || audio.length !== (silence ? 1 : 0)) {- throw new Error("Stand-in video differs from its requested frame domain");- }- return await env.putFile(output, "video/mp4");- } finally {- await rm(work, { recursive: true, force: true }).catch(() => {});- }+ return await env.putBytes(drawStandInCard(request), "image/png");}diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.tsindex 90611c0a..b4aa8865 100644--- a/packages/media-execution/src/execute.ts+++ b/packages/media-execution/src/execute.ts@@ -32,8 +32,7 @@ import {} from "@hypit/protocol";import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";-import { drawStandInSilence, renderStandInCard, standInCardNeed } from "./card.js";-import { verifyStandInSilenceRequest } from "@hypit/stand-in";+import { drawClipTimeGuide, renderStandInCard, standInCardNeed } from "./card.js";import { parseMediaInspection } from "./probe.js";import {compositeAnimatedWebpFrame,@@ -772,7 +771,7 @@ export async function executeNormalizeMedia(}/**- * Encode authored images into an exact silent CFR video. One picture is held for the whole frame+ * Encode authored images into an exact video-only CFR clip. One picture is held for the whole frame* count; several are each held for their planned segment, fitted into the first picture's frame* and letterboxed on black, then concatenated in order.*/@@ -784,6 +783,7 @@ export async function executeRenderStillVideo(const work = await mkdtemp(join(tmpdir(), "hypit-media-still-"));try {const output = join(work, "still.mp4");+ const baseOutput = need.request.guide === undefined ? output : join(work, "still-base.mp4");const staged = new Map<string, string>();const inputs: string[] = [];for (const segment of need.request.segments) {@@ -833,12 +833,47 @@ export async function executeRenderStillVideo(...argv,"-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr","-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",- "-movflags", "+faststart", output,+ "-movflags", "+faststart", baseOutput,],timeoutMs: env.processTimeoutMs,maxStdoutBytes: 64 * 1024,...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),});+ if (need.request.guide === "clip-time") {+ const baseInspection = await outputInspection({+ path: baseOutput,+ mediaType: "video/mp4",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ const baseVisual = baseInspection.streams.find((item): item is MediaVideoStream => item.kind === "video");+ assert(baseVisual !== undefined, "Still media guide has no visual stream");+ for (let frame = 0; frame < need.request.frameCount; frame += 1) {+ await writeFile(join(work, `guide-${String(frame).padStart(6, "0")}.png`), drawClipTimeGuide({+ width: baseVisual.width,+ height: baseVisual.height,+ frameRate: need.request.frameRate,+ frameCount: need.request.frameCount,+ }, frame));+ }+ await runProcess({+ executable: env.ffmpegPath,+ argv: [+ "-y", "-i", baseOutput,+ "-framerate", fps, "-i", join(work, "guide-%06d.png"),+ "-filter_complex", "[0:v][1:v]overlay=0:main_h-overlay_h:shortest=1,format=yuv420p[v]",+ "-map", "[v]", "-an",+ "-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr",+ "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",+ "-movflags", "+faststart", output,+ ],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ }const inspected = await outputInspection({path: output,mediaType: "video/mp4",@@ -872,43 +907,11 @@ export async function executeDrawStandInCard(): Promise<MediaOperationResult> {const request = standInCardNeed(constraints);const value = await renderStandInCard({- ffmpegPath: env.ffmpegPath,- processTimeoutMs: env.processTimeoutMs,- runFfmpeg: async (argv) => {- await runProcess({- executable: env.ffmpegPath,- argv,- timeoutMs: env.processTimeoutMs,- maxStdoutBytes: 64 * 1024,- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),- });- },- inspectStreams: async (path) => {- const inspected = await outputInspection({- path,- mediaType: "video/mp4",- ffprobePath: env.ffprobePath,- timeoutMs: env.processTimeoutMs,- maxProbeOutputBytes: env.maxProbeOutputBytes,- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),- });- return inspected.streams;- },putBytes: async (bytes, mediaType) => await env.artifacts.put(bytes, mediaType) as unknown as CanonicalValue,- putFile: async (path, mediaType) => await env.artifacts.putFile(path, mediaType) as unknown as CanonicalValue,}, request);return artifactResult(value as unknown as BlobRef);}-/** Materialize the generic silent-audio Candidate selected by a Run. */-export async function executeDrawStandInSilence(- env: MediaExecutionEnvironment,- constraints: CanonicalValue,-): Promise<MediaOperationResult> {- verifyStandInSilenceRequest(constraints);- return artifactResult(await env.artifacts.put(drawStandInSilence(constraints.sampleFrames), "audio/wav"));-}-type TransformPlan = {readonly videoFilters: readonly string[];readonly audioFilters: readonly string[];diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.tsindex 04bd8007..ac3fa63b 100644--- a/packages/media-execution/src/index.ts+++ b/packages/media-execution/src/index.ts@@ -8,10 +8,10 @@ export {executeRenderTimelineAudio,executeRenderStillVideo,executeDrawStandInCard,- executeDrawStandInSilence,executeTransformMedia,} from "./execute.js";-export { drawStandInBand, drawStandInCard, drawStandInSilence } from "./card.js";+export { drawClipTimeGuide, drawStandInCard } from "./card.js";+export type { ClipTimeGuideRequest } from "./card.js";export { parseMediaInspection } from "./probe.js";export { verifyCompositableSurfaceBytes } from "./surface.js";export { probeMediaToolchain } from "./toolchain.js";diff --git a/packages/media-pipeline/README.md b/packages/media-pipeline/README.mdindex 1f9b4447..794773ba 100644--- a/packages/media-pipeline/README.md+++ b/packages/media-pipeline/README.md@@ -67,7 +67,11 @@ Several images spread over one literal duration the same way, each held for its</media:StillVideo>```-`StillVideo` returns a silent MP4 `BlobArtifact`, not `SynchronizedMedia`. The Surface publishes the+For a short-lived diagnostic clip, `guide="clip-time"` burns one coherent band containing the+clip-local timecode, current frame and progress ruler. Omit it for clean production pixels. The+guide says nothing about the clip's eventual position in a Film.++`StillVideo` returns a video-only MP4 `BlobArtifact`, not `SynchronizedMedia`. The Surface publishes theduration and the weights as Records; `plan-still-video` divides the whole frame count among thepictures (every picture holds at least one frame, the rest go by weight with leftovers to the largestremainders, earlier first), `bind-still-video-source` attaches each picture in authored order, and@@ -76,10 +80,10 @@ frame and letterboxed. Inspection and normalization remain the same explicit steor generated moving video, so the Blob can serve B-roll through Normalize, or continue into eitherestimated or measured A-roll semantics afterward.-The package also exposes its one-picture Fragment as-`@hypit/media-pipeline@1#still-video`, so a Run may select the same ordinary-StillVideo branch as a Candidate. Its inputs are `duration`, `clock`, `layout`-and `source-0`; choosing it remains an explicit `candidate` / `satisfy` decision.+The package also exposes one-picture Run Fragments. `still-video` takes `duration`, `clock`,+`layout` and `source-0`; `clip-time-still-video` takes `duration`, `clock` and `source`, supplying+the diagnostic layout itself. A Run can select either ordinary branch as a Candidate through an+explicit `candidate` / `satisfy` decision.Four ordinary author operations reuse that same inspection/execution boundary:diff --git a/packages/media-pipeline/src/activation.ts b/packages/media-pipeline/src/activation.tsindex 18524f95..15c9d7e4 100644--- a/packages/media-pipeline/src/activation.ts+++ b/packages/media-pipeline/src/activation.ts@@ -11,6 +11,7 @@ import {mediaPipelineModuleRef,mediaPipelineMarkupSurfaces,stillVideoFragment,+ clipTimeStillVideoFragment,} from "./index.js";export const hypitPackage = {@@ -19,7 +20,10 @@ export const hypitPackage = {components: [mediaPipelineComponent],hostFacets: [createRunFragmentHostFacet({name: "@hypit/media-pipeline@1",- fragments: { "still-video": stillVideoFragment },+ fragments: {+ "still-video": stillVideoFragment,+ "clip-time-still-video": clipTimeStillVideoFragment,+ },}), createMarkupSurfaceHostFacet({module: mediaPipelineModuleRef,declaration: mediaPipelineMarkupSurfaces.find((item) => item.name === "synchronized-media")!,diff --git a/packages/media-pipeline/src/component.ts b/packages/media-pipeline/src/component.tsindex 47febd12..3f5afa76 100644--- a/packages/media-pipeline/src/component.ts+++ b/packages/media-pipeline/src/component.ts@@ -229,6 +229,15 @@ export const mediaPipelineComponent = {return { outputs: {}, needs: { image: canonicalize(need) } };},},+ {+ producer: mediaPipelineProducers.clipTimeLayout,+ handler: () => ({+ outputs: {+ layout: { kind: "inline", value: canonicalize({ weights: [1], guide: "clip-time" }) },+ },+ needs: {},+ }),+ },{producer: mediaPipelineProducers.planStill,handler: ({ inputs }) => {@@ -245,6 +254,7 @@ export const mediaPipelineComponent = {const request = sealStillVideoRequest({frameRate: clock.frameRate,frameCount: frames,+ ...(layout.guide === undefined ? {} : { guide: layout.guide }),output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },segments: planStillVideoSegments(frames, layout.weights),});diff --git a/packages/media-pipeline/src/fragment.ts b/packages/media-pipeline/src/fragment.tsindex 18651440..064303b7 100644--- a/packages/media-pipeline/src/fragment.ts+++ b/packages/media-pipeline/src/fragment.ts@@ -165,3 +165,39 @@ export function createStillVideoFragment(count: number) {}export const stillVideoFragment = createStillVideoFragment(1);++/** One picture held with a clip-local timecode, frame count and progress ruler for diagnosis. */+export const clipTimeStillVideoFragment = sealGraphFragment({+ inputs: [+ { name: "duration", type: speechTypes.duration },+ { name: "clock", type: programSpaceTypes.clock },+ { name: "source", type: artifactTypes.blob },+ ],+ operations: [+ {+ id: "layout",+ producer: mediaPipelineProducers.clipTimeLayout,+ inputs: {},+ result: { kind: "output", name: "layout" },+ },+ {+ id: "plan",+ producer: mediaPipelineProducers.planStill,+ inputs: { duration: input("duration"), clock: input("clock"), layout: operation("layout") },+ result: { kind: "output", name: "request" },+ },+ {+ id: "bind",+ producer: mediaPipelineProducers.bindStill,+ inputs: { request: operation("plan"), source: input("source") },+ result: { kind: "output", name: "request" },+ },+ {+ id: "render",+ producer: mediaPipelineProducers.renderStill,+ inputs: { request: operation("bind") },+ result: { kind: "need", name: "video" },+ },+ ],+ exports: [{ name: "video", type: artifactTypes.blob, root: operation("render") }],+});diff --git a/packages/media-pipeline/src/index.ts b/packages/media-pipeline/src/index.tsindex 5439410d..f64d7a1c 100644--- a/packages/media-pipeline/src/index.ts+++ b/packages/media-pipeline/src/index.ts@@ -7,6 +7,7 @@ export {extractFrameFragment,synchronizedMediaFragment,stillVideoFragment,+ clipTimeStillVideoFragment,createStillVideoFragment,transformMediaFragment,} from "./fragment.js";@@ -15,7 +16,7 @@ export {sealAudioProgramPlan,verifyAudioProgramPlan,} from "./audio-plan.js";-export { mediaPipelineCapabilities, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, mediaTransformProgramSchema } from "./manifest.js";+export { mediaPipelineCapabilities, mediaPipelineManifest, mediaPipelineMarkupSurfaces, mediaPipelineModuleRef, mediaPipelineProducers, mediaPipelineTypes, audioProgramPlanSchema, mediaSelectionRequestSchema, audioExtractionRequestSchema, frameExtractionRequestSchema, stillVideoRequestSchema, stillVideoLayoutSchema, mediaTransformProgramSchema } from "./manifest.js";export {sealMediaSelectionRequest,selectMediaStreams,diff --git a/packages/media-pipeline/src/manifest.ts b/packages/media-pipeline/src/manifest.tsindex ec9226af..8aa9b3f9 100644--- a/packages/media-pipeline/src/manifest.ts+++ b/packages/media-pipeline/src/manifest.ts@@ -41,6 +41,7 @@ export const mediaPipelineProducers = {transform: { module: mediaPipelineModuleRef, name: "request-media-transform" },extractAudio: { module: mediaPipelineModuleRef, name: "request-audio-extraction" },extractFrame: { module: mediaPipelineModuleRef, name: "request-frame-extraction" },+ clipTimeLayout: { module: mediaPipelineModuleRef, name: "clip-time-still-layout" },planStill: { module: mediaPipelineModuleRef, name: "plan-still-video" },renderStill: { module: mediaPipelineModuleRef, name: "request-still-video" },bindStill: { module: mediaPipelineModuleRef, name: "bind-still-video-source" },@@ -163,6 +164,7 @@ export const stillVideoRequestSchema: ValueSchema = {},} },frameCount: { schema: { kind: "number", integer: true, minimum: 1 } },+ guide: { optional: true, schema: { kind: "literal", value: "clip-time" } },output: { schema: {kind: "object",fields: {@@ -194,6 +196,7 @@ export const stillVideoLayoutSchema: ValueSchema = {kind: "object",fields: {weights: { schema: { kind: "array", items: { kind: "number", minimum: 0 } } },+ guide: { optional: true, schema: { kind: "literal", value: "clip-time" } },},};@@ -293,7 +296,7 @@ export const mediaPipelineMarkupSurfaces = [name: "still-video", tag: "StillVideo", mode: "structured",outputs: [speechTypes.duration, mediaPipelineTypes.stillVideoLayout, mediaPipelineTypes.stillVideoRequest, artifactTypes.blob],vocabulary: {- summary: "Spreads one or more authored images over an explicit duration as an ordinary silent MP4 Blob on a frame clock.",+ summary: "Spreads one or more authored images over an explicit duration as an ordinary video-only MP4 Blob on a frame clock.",attributes: [{ name: "id", kind: "identifier", required: true,summary: "Names the still-video operation and the MP4 Artifact it publishes." },@@ -303,6 +306,8 @@ export const mediaPipelineMarkupSurfaces = [summary: "Sets the video's length in seconds, such as 6 or 2.5s; the author's decision, measured beforehand with hypit measure." },{ name: "clock", kind: "reference", required: true, accepts: [programSpaceTypes.clock],summary: "Selects the frame clock used by the generated MP4." },+ { name: "guide", kind: "literal", required: false, values: ["clip-time"],+ summary: "Burns one clip-local diagnostic band with timecode, frame count and a progress ruler into the held video." },],children: [{ tag: "Still", cardinality: "many",@@ -315,7 +320,7 @@ export const mediaPipelineMarkupSurfaces = [] },],ports: [{ name: "video", type: artifactTypes.blob,- summary: "The ordinary silent MP4 Artifact, addressed as `<id>.video`." }],+ summary: "The ordinary video-only MP4 Artifact, addressed as `<id>.video`." }],example: `<media:StillVideo id="kitchen-stills" duration="6" clock={clock}><media:Still source={counter}/><media:Still source={basil} weight="2"/>@@ -325,6 +330,7 @@ export const mediaPipelineMarkupSurfaces = ["Write exactly one of source or Still children. Frames are whole: each image gets the floor of its share and the leftover frames go to the largest remainders, so the split is deterministic and every image holds at least one frame.","Images of different sizes are fitted into the first image's frame, letterboxed on black.","The result is a normal video Blob, not SynchronizedMedia and not a SemanticTake.",+ "The optional clip-time guide describes this clip's local frame domain, not its later position in a Film.","Use Normalize afterward exactly as for generated or imported moving video.","Encoding is a render-still-video Need fulfilled by the selected media Provider; this Surface never invokes FFmpeg itself.",],@@ -560,6 +566,12 @@ export const mediaPipelineManifest: ModuleManifest = {returns: artifactTypes.blob,}],},+ {+ name: mediaPipelineProducers.clipTimeLayout.name,+ inputs: [],+ outputs: [{ name: "layout", type: mediaPipelineTypes.stillVideoLayout }],+ needs: [],+ },{name: mediaPipelineProducers.planStill.name,inputs: [diff --git a/packages/media-pipeline/src/operations.ts b/packages/media-pipeline/src/operations.tsindex ad14e19c..7d48fcf4 100644--- a/packages/media-pipeline/src/operations.ts+++ b/packages/media-pipeline/src/operations.ts@@ -123,6 +123,7 @@ export function verifyStillVideoLayout(value: unknown): asserts value is StillVifor (const weight of layout.weights) {assert(typeof weight === "number" && Number.isFinite(weight) && weight > 0, "StillVideoLayout weights must be positive");}+ assert(layout.guide === undefined || layout.guide === "clip-time", "StillVideoLayout guide is unsupported");}export function sealStillVideoLayout(value: StillVideoLayout): StillVideoLayout {@@ -170,6 +171,7 @@ export function verifyStillVideoRequest(value: unknown): asserts value is StillV"StillVideoRequest.frameRate must be a positive rational");assert(Number.isSafeInteger(request.frameCount) && (request.frameCount as number) > 0,"StillVideoRequest.frameCount must be positive");+ assert(request.guide === undefined || request.guide === "clip-time", "StillVideoRequest guide is unsupported");const output = object(request.output, "StillVideoRequest.output");assert(output.container === "mp4" && output.codec === "h264" && output.pixelFormat === "yuv420p","StillVideoRequest output profile is unsupported");diff --git a/packages/media-pipeline/src/surface.ts b/packages/media-pipeline/src/surface.tsindex c1e94e06..5310e804 100644--- a/packages/media-pipeline/src/surface.ts+++ b/packages/media-pipeline/src/surface.ts@@ -258,7 +258,7 @@ function stillPictures(}export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {- exactAttributes(element, ["id", "duration", "clock"], ["source"]);+ exactAttributes(element, ["id", "duration", "clock"], ["source", "guide"]);const id = text(element, "id");const pictures = stillPictures(element, resolveReference);// The duration is the author's decision, written here; it is published as an ordinary@@ -266,6 +266,10 @@ export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, resconst durationSec = seconds(text(element, "duration"), `${element.name}.duration`, false);const durationId = `${id}.duration`;const layoutId = `${id}.layout`;+ const guide = element.attributes.guide;+ if (guide !== undefined && guide !== "clip-time") {+ throw new Error(`${element.name}.guide must be clip-time when written.`);+ }const clock = typedReference(element.attributes.clock,`${element.name}.clock`,@@ -282,7 +286,10 @@ export const decodeStillVideoSurface: StructuredSurfaceHandler = ({ element, res}, {id: layoutId,type: mediaPipelineTypes.stillVideoLayout,- value: { kind: "inline", value: { weights: pictures.map((picture) => picture.weight) } },+ value: { kind: "inline", value: {+ weights: pictures.map((picture) => picture.weight),+ ...(guide === undefined ? {} : { guide }),+ } },range: element.range,}],components: [{diff --git a/packages/media-pipeline/src/types.ts b/packages/media-pipeline/src/types.tsindex 888f315f..f9374fd0 100644--- a/packages/media-pipeline/src/types.ts+++ b/packages/media-pipeline/src/types.ts@@ -70,6 +70,8 @@ export type FrameExtractionRequest = {/** How the still video divides its literal duration among its pictures: one weight per picture, in order. */export type StillVideoLayout = {readonly weights: readonly number[];+ /** Optional diagnostic guide burned into the held clip; omitted keeps production pixels clean. */+ readonly guide?: "clip-time";};/** One picture's span of the still video, in frames; `source` is bound one picture at a time. */@@ -79,10 +81,11 @@ export type StillVideoSegment = {readonly source?: BlobRef;};-/** Exact silent video requested from one or more authored still images spread over a literal duration. */+/** Exact video-only clip requested from one or more authored still images spread over a literal duration. */export type StillVideoRequest = {readonly frameRate: MediaRational;readonly frameCount: number;+ readonly guide?: "clip-time";readonly output: {readonly container: "mp4";readonly codec: "h264";diff --git a/packages/provider-media-local/README.md b/packages/provider-media-local/README.mdindex 85156a3b..a2736cae 100644--- a/packages/provider-media-local/README.md+++ b/packages/provider-media-local/README.md@@ -22,7 +22,7 @@ The Endpoint:- preserves source A/V offset through deterministic trim, delay, pad and crop operations;- emits a silent CFR visual and, when selected, an exact-length 48 kHz stereo PCM WAV;- transforms synchronized A/V, extracts generic reference audio and extracts exact source frames;-- encodes one authored image frame as an ordinary finite silent MP4 before normalization;+- encodes one authored image frame as an ordinary finite video-only MP4 before normalization;- renders an explicit `AudioProgramPlan` into one exact-length 48 kHz stereo PCM `TimelineAudio`;- muxes exactly one verified silent visual stream and one verified program-audio stream into MP4;- distinguishes AAC coding-frame padding from the authoritative packet presentation span.diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.tsindex b68a53b7..9fa69c09 100644--- a/packages/provider-media-local/src/provider.ts+++ b/packages/provider-media-local/src/provider.ts@@ -10,7 +10,6 @@ import {executeRenderTimelineAudio,executeRenderStillVideo,executeDrawStandInCard,- executeDrawStandInSilence,executeTransformMedia,} from "@hypit/media-execution";import type { MediaExecutionEnvironment, MediaOperationResult } from "@hypit/media-execution";@@ -135,13 +134,6 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptionsreturns: artifactTypes.blob,handler: operation(executeDrawStandInCard),},- {- lifecycle: "immediate" as const,- transient: true,- capability: standInCapabilities.drawSilence,- returns: artifactTypes.blob,- handler: operation(executeDrawStandInSilence),- },{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.tsindex e089f08f..10ebdcd0 100644--- a/packages/provider-media-local/test/provider.test.ts+++ b/packages/provider-media-local/test/provider.test.ts@@ -198,7 +198,6 @@ test("local media opts only reusable display materialization into transient exec"extract-media-frame","render-still-video","draw-card",- "draw-silence",]);assert.deepEqual(provider.offers.filter((offer) => offer.transient !== true).map((offer) => offer.capability.name), [@@ -518,7 +517,7 @@ test("local media Provider preserves one source A/V origin when audio starts lat}});-test("a silent generated MP4 remains a visual-only product and cannot satisfy a requested audio stream", {+test("a video-only generated MP4 remains visual-only and cannot satisfy a requested audio stream", {skip: !hasMediaBinaries,}, async () => {const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-silent-"));@@ -644,6 +643,7 @@ test("local media Provider transforms A/V and extracts ordinary audio and framerequest: {frameRate: { numerator: 30, denominator: 1 },frameCount: 15,+ guide: "clip-time",output: { container: "mp4", codec: "h264", pixelFormat: "yuv420p" },segments: [{ startFrame: 0, endFrameExclusive: 15, source: extractedFrame }],},@@ -694,7 +694,7 @@ test("local media Provider transforms A/V and extracts ordinary audio and frame}});-test("local media Provider renders one frame-domain audio plan and muxes exactly one silent visual with it", {+test("local media Provider renders one frame-domain audio plan and muxes exactly one video-only visual with it", {skip: !hasMediaBinaries,}, async () => {const root = await mkdtemp(join(tmpdir(), "hypit-provider-media-program-"));@@ -876,9 +876,7 @@ function pngSize(bytes: Uint8Array): { readonly width: number; readonly height:return { width: view.getUint32(16), height: view.getUint32(20) };}-test("local media Provider draws a stand-in card as a picture and as an exact silent clip", {- skip: !hasMediaBinaries && "ffmpeg is not installed",-}, async () => {+test("local media Provider draws a stand-in Card image", async () => {const resources = new MemoryResourceStore();const fulfillBlob = async (request: Need): Promise<Awaited<ReturnType<MemoryResourceStore["put"]>>> => {const provider = await handlerFor(request);@@ -892,42 +890,13 @@ test("local media Provider draws a stand-in card as a picture and as an exact sireturn result.value as Awaited<ReturnType<MemoryResourceStore["put"]>>;};const picture = await fulfillBlob(need("need:card-image", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({- kind: "image", width: 640, height: 400,+ width: 640, height: 400,})));assert.equal(picture.kind, "blob");assert.equal(picture.mediaType, "image/png");const bytes = await resources.get(picture.resource);assert.ok(bytes !== undefined);assert.deepEqual(pngSize(bytes), { width: 640, height: 400 });-- const clip = await fulfillBlob(need("need:card-video", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({- kind: "video", width: 320, height: 568,- video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12 },- })));- assert.equal(clip.mediaType, "video/mp4");- const inspection = await inspectArtifact(resources, clip);- const visual = inspection.streams.find((item) => item.kind === "video");- assert.ok(visual !== undefined && visual.kind === "video");- assert.equal(inspection.streams.length, 1, "the card request alone decides whether a clip carries silence");- assert.equal(visual.width, 320);- assert.equal(visual.height, 568);- assert.equal(visual.decodedUnitCount, 12);-- const spoken = await fulfillBlob(need("need:card-spoken", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({- kind: "video", width: 320, height: 568,- video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12, audio: "silence" },- })));- const spokenStreams = (await inspectArtifact(resources, spoken)).streams;- const sound = spokenStreams.find((item) => item.kind === "audio");- assert.ok(sound !== undefined && sound.kind === "audio", "a generic video stand-in can carry a silent track");- assert.equal(sound.sampleRate, 48_000);- assert.equal(spokenStreams.length, 2);-- const silence = await fulfillBlob(need("need:stand-in-silence", standInCapabilities.drawSilence, artifactTypes.blob,- canonicalize({ sampleRate: 48_000, channels: 2, sampleFrames: 48_000 })));- assert.equal(silence.mediaType, "audio/wav");- const silenceBytes = await resources.get(silence.resource);- assert.equal(silenceBytes?.byteLength, 44 + 48_000 * 4);});diff --git a/packages/stand-in/README.md b/packages/stand-in/README.mdindex d3ebfd84..fc822a4f 100644--- a/packages/stand-in/README.md+++ b/packages/stand-in/README.md@@ -1,16 +1,18 @@# `@hypit/stand-in`-A stand-in is an ordinary generic Candidate selected by a Run in place of an output that is not-needed yet. It knows nothing about the displaced Candidate's Producer, model, prompt or request.-The Run supplies only the media shape the downstream composition needs, and the resulting file goes-through the same inspect and normalize steps as any other file.+A stand-in Card is an ordinary generic image Candidate selected by a Run when a composition needs+visible pixels before the intended media exists. It knows nothing about the displaced Candidate's+Producer, model, prompt or request. The resulting file goes through the same media operations as any+other image.-This package exports three Run Fragments: `image`, `video` and `silence`. A local media Provider-materializes their deterministic files. Model packages do not know these Fragments exist.+This package exports `card` and the convenience composition `timed-card`. A local media Provider+draws the Card. `timed-card` then gives that image to `@hypit/media-pipeline`, which holds it for the+requested duration and adds its clip-local time guide. Model packages do not know these+Fragments exist.```xml<import from="@hypit/stand-in@1" as="stand-in"/>-<fragment id="kitchen-card" using="stand-in:video">+<fragment id="kitchen-card" using="stand-in:timed-card"><input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/>@@ -18,20 +20,19 @@ materializes their deterministic files. Model packages do not know these Fragmen<satisfy output="kitchen.video" candidate="kitchen-card.video"/>```-Remove the `satisfy` line to select the Source's primary Candidate again. The card carries a-diagonal STAND-IN watermark, its kind, frame and duration and — for video — a running timecode and-progress bar, so a cut that lands on the wrong frame remains visible.+Remove the `satisfy` line to select the Source's primary Candidate again. The Card has one visible+label and its canvas dimensions. The timed composition adds a coherent timecode, frame count and+progress guide, so the exact clip-local frame remains visible.## Fragment inputs and exports| Fragment | Inputs | Export || --- | --- | --- |-| `image` | `canvas`: CanvasSpace | `image`: BlobArtifact |-| `video` | `canvas`: CanvasSpace, `duration`: SpeechDuration in seconds, `clock`: ProgramClock | `video`: BlobArtifact |-| `silence` | `duration`: SpeechDuration in seconds | `audio`: BlobArtifact |+| `card` | `canvas`: CanvasSpace | `image`: BlobArtifact |+| `timed-card` | `canvas`: CanvasSpace, `duration`: SpeechDuration in seconds, `clock`: ProgramClock | `video`: BlobArtifact |`from` on a Run input refers to a public value of the Author entry; literal duration can use-`value="5"`. The card/silence capability still needs a selected media Provider. Studio can use it+`value="5"`. The card capability still needs a selected media Provider. Studio can use itwhen that Provider admits it for transient execution. An estimated SemanticTake is a separatechoice supplied by `@hypit/semantic-take-estimate` after normalization.diff --git a/packages/stand-in/package.json b/packages/stand-in/package.jsonindex 4b95a955..9c3dd841 100644--- a/packages/stand-in/package.json+++ b/packages/stand-in/package.json@@ -14,7 +14,7 @@"@hypit/artifact": "workspace:*","@hypit/component-kit": "workspace:*","@hypit/elaborator": "workspace:*",- "@hypit/media": "workspace:*",+ "@hypit/media-pipeline": "workspace:*","@hypit/program-space": "workspace:*","@hypit/protocol": "workspace:*","@hypit/run": "workspace:*",diff --git a/packages/stand-in/src/activation.ts b/packages/stand-in/src/activation.tsindex ee16c204..31579be9 100644--- a/packages/stand-in/src/activation.ts+++ b/packages/stand-in/src/activation.ts@@ -1,6 +1,6 @@import { standInComponent, standInManifest } from "./index.js";import { createRunFragmentHostFacet } from "@hypit/run";-import { standInImageFragment, standInSilenceFragment, standInVideoFragment } from "./fragment.js";+import { standInCardFragment, standInTimedCardFragment } from "./fragment.js";export const hypitPackage = {format: "hypit.node-package@1" as const,@@ -9,9 +9,8 @@ export const hypitPackage = {hostFacets: [createRunFragmentHostFacet({name: "@hypit/stand-in@1",fragments: {- image: standInImageFragment,- video: standInVideoFragment,- silence: standInSilenceFragment,+ card: standInCardFragment,+ "timed-card": standInTimedCardFragment,},})],};diff --git a/packages/stand-in/src/fragment.ts b/packages/stand-in/src/fragment.tsindex 2d00d022..1bbc409e 100644--- a/packages/stand-in/src/fragment.ts+++ b/packages/stand-in/src/fragment.ts@@ -1,5 +1,6 @@import { artifactTypes } from "@hypit/artifact";import { sealGraphFragment } from "@hypit/elaborator";+import { mediaPipelineProducers } from "@hypit/media-pipeline";import { programSpaceTypes } from "@hypit/program-space";import { spatialTypes } from "@hypit/spatial";import { speechTypes } from "@hypit/speech";@@ -9,39 +10,55 @@ import { standInProducers } from "./manifest.js";const input = (name: string) => ({ kind: "fragment-input" as const, name });const operation = (name: string) => ({ kind: "fragment-operation" as const, operation: name });-export const standInImageFragment = sealGraphFragment({+export const standInCardFragment = sealGraphFragment({inputs: [{ name: "canvas", type: spatialTypes.canvas }],operations: [{- id: "image",- producer: standInProducers.image,+ id: "card",+ producer: standInProducers.card,inputs: { canvas: input("canvas") },result: { kind: "need", name: "image" },}],- exports: [{ name: "image", type: artifactTypes.blob, root: operation("image") }],+ exports: [{ name: "image", type: artifactTypes.blob, root: operation("card") }],});-export const standInVideoFragment = sealGraphFragment({+/** A Card held by media-pipeline with its optional clip-local diagnostic guide. */+export const standInTimedCardFragment = sealGraphFragment({inputs: [{ name: "canvas", type: spatialTypes.canvas },{ name: "duration", type: speechTypes.duration },{ name: "clock", type: programSpaceTypes.clock },],- operations: [{- id: "video",- producer: standInProducers.video,- inputs: { canvas: input("canvas"), duration: input("duration"), clock: input("clock") },- result: { kind: "need", name: "video" },- }],- exports: [{ name: "video", type: artifactTypes.blob, root: operation("video") }],-});--export const standInSilenceFragment = sealGraphFragment({- inputs: [{ name: "duration", type: speechTypes.duration }],- operations: [{- id: "silence",- producer: standInProducers.silence,- inputs: { duration: input("duration") },- result: { kind: "need", name: "audio" },- }],- exports: [{ name: "audio", type: artifactTypes.blob, root: operation("silence") }],+ operations: [+ {+ id: "card",+ producer: standInProducers.card,+ inputs: { canvas: input("canvas") },+ result: { kind: "need", name: "image" },+ },+ {+ id: "layout",+ producer: mediaPipelineProducers.clipTimeLayout,+ inputs: {},+ result: { kind: "output", name: "layout" },+ },+ {+ id: "plan",+ producer: mediaPipelineProducers.planStill,+ inputs: { duration: input("duration"), clock: input("clock"), layout: operation("layout") },+ result: { kind: "output", name: "request" },+ },+ {+ id: "bind",+ producer: mediaPipelineProducers.bindStill,+ inputs: { request: operation("plan"), source: operation("card") },+ result: { kind: "output", name: "request" },+ },+ {+ id: "render",+ producer: mediaPipelineProducers.renderStill,+ inputs: { request: operation("bind") },+ result: { kind: "need", name: "video" },+ },+ ],+ exports: [{ name: "video", type: artifactTypes.blob, root: operation("render") }],});diff --git a/packages/stand-in/src/index.ts b/packages/stand-in/src/index.tsindex 4b0bfecc..ad99b997 100644--- a/packages/stand-in/src/index.ts+++ b/packages/stand-in/src/index.ts@@ -1,13 +1,8 @@import type { ComponentPackage } from "@hypit/component-kit";-import type { MediaRational } from "@hypit/media";-import { assertProgramClockIdentity } from "@hypit/program-space";-import type { ProgramClock } from "@hypit/program-space";import { canonicalize } from "@hypit/protocol";import type { CanonicalValue, StoredValue } from "@hypit/protocol";import { assertCanvasSpace } from "@hypit/spatial";import type { CanvasSpace } from "@hypit/spatial";-import { assertSpeechDurationIdentity } from "@hypit/speech";-import type { SpeechDuration } from "@hypit/speech";import { standInProducers, standInTypes } from "./manifest.js";@@ -19,69 +14,33 @@ export {standInProducers,standInTypes,} from "./manifest.js";-export { standInImageFragment, standInSilenceFragment, standInVideoFragment } from "./fragment.js";+export { standInCardFragment, standInTimedCardFragment } from "./fragment.js";/*** The minimum visible shape of a generic stand-in. It says nothing about the* Producer or model whose output a Run may choose to replace with it.*/export type StandInCardRequest = {- readonly kind: "image" | "video";- /** Even, positive pixel dimensions of the card, the frame the real output would have. */+ /** Positive pixel dimensions of the Card. */readonly width: number;readonly height: number;- /** A video card runs for exactly this many frames at this clock and carries a running timecode. */- readonly video?: {- readonly frameRate: MediaRational;- readonly frameCount: number;- /** The model would have generated sound, so the card carries a silent 48 kHz stereo track. */- readonly audio?: "silence";- };-};--export type StandInSilenceRequest = {- readonly sampleRate: 48_000;- readonly channels: 2;- readonly sampleFrames: number;};function assert(condition: unknown, message: string): asserts condition {if (!condition) throw new Error(message);}-function evenPositive(value: unknown, subject: string): number {- assert(typeof value === "number" && Number.isSafeInteger(value) && value > 0 && value % 2 === 0,- `${subject} must be a positive even integer`);+function positiveInteger(value: unknown, subject: string): number {+ assert(typeof value === "number" && Number.isSafeInteger(value) && value > 0,+ `${subject} must be a positive integer`);return value;}export function verifyStandInCardRequest(value: unknown): asserts value is StandInCardRequest {assert(value !== null && typeof value === "object" && !Array.isArray(value), "StandInCardRequest must be an object");const item = value as Record<string, unknown>;- assert(item.kind === "image" || item.kind === "video", "StandInCardRequest kind must be image or video");- evenPositive(item.width, "StandInCardRequest width");- evenPositive(item.height, "StandInCardRequest height");- if (item.kind === "video") {- const video = item.video as Record<string, unknown> | undefined;- assert(video !== undefined && typeof video === "object", "a video StandInCardRequest needs its frame domain");- const rate = video.frameRate as Record<string, unknown> | undefined;- assert(rate !== undefined && Number.isSafeInteger(rate.numerator) && (rate.numerator as number) > 0- && Number.isSafeInteger(rate.denominator) && (rate.denominator as number) > 0,- "StandInCardRequest frameRate must be a positive rational");- assert(Number.isSafeInteger(video.frameCount) && (video.frameCount as number) > 0,- "StandInCardRequest frameCount must be a positive integer");- assert(video.audio === undefined || video.audio === "silence", "StandInCardRequest audio can only be silence");- } else {- assert(item.video === undefined, "an image StandInCardRequest has no frame domain");- }-}--export function verifyStandInSilenceRequest(value: unknown): asserts value is StandInSilenceRequest {- assert(value !== null && typeof value === "object" && !Array.isArray(value), "StandInSilenceRequest must be an object");- const item = value as Record<string, unknown>;- assert(item.sampleRate === 48_000 && item.channels === 2, "StandInSilenceRequest must be 48 kHz stereo");- assert(Number.isSafeInteger(item.sampleFrames) && (item.sampleFrames as number) > 0,- "StandInSilenceRequest sampleFrames must be a positive integer");+ positiveInteger(item.width, "StandInCardRequest width");+ positiveInteger(item.height, "StandInCardRequest height");}export function sealStandInCardRequest(value: StandInCardRequest): StandInCardRequest {@@ -98,63 +57,23 @@ function typedInline<T>(value: StoredValue, subject: string): T {return inline(value, subject) as T;}-function exactFrameCount(duration: SpeechDuration, clock: ProgramClock): number {- const frames = duration * clock.frameRate.numerator / clock.frameRate.denominator;- const rounded = Math.round(frames);- assert(Number.isSafeInteger(rounded) && rounded > 0 && Math.abs(frames - rounded) < 1e-7,- "Stand-in video duration must end on a whole ProgramClock frame");- return rounded;-}-export const standInComponent = {validators: [{type: standInTypes.cardRequest,handler: ({ value }) => { verifyStandInCardRequest(inline(value, "StandInCardRequest")); },},- {- type: standInTypes.silenceRequest,- handler: ({ value }) => { verifyStandInSilenceRequest(inline(value, "StandInSilenceRequest")); },- },],producers: [{- producer: standInProducers.image,+ producer: standInProducers.card,handler: ({ inputs }) => {- const canvas = typedInline<CanvasSpace>(inputs.canvas!.value, "stand-in image Canvas");+ const canvas = typedInline<CanvasSpace>(inputs.canvas!.value, "stand-in Canvas");assertCanvasSpace(canvas);return { outputs: {}, needs: { image: canonicalize(sealStandInCardRequest({- kind: "image", width: canvas.widthPx, height: canvas.heightPx,+ width: canvas.widthPx, height: canvas.heightPx,})) } };},},- {- producer: standInProducers.video,- handler: ({ inputs }) => {- const canvas = typedInline<CanvasSpace>(inputs.canvas!.value, "stand-in video Canvas");- const duration = typedInline<SpeechDuration>(inputs.duration!.value, "stand-in video duration");- const clock = typedInline<ProgramClock>(inputs.clock!.value, "stand-in video clock");- assertCanvasSpace(canvas);- assertSpeechDurationIdentity(duration);- assertProgramClockIdentity(clock);- return { outputs: {}, needs: { video: canonicalize(sealStandInCardRequest({- kind: "video",- width: canvas.widthPx,- height: canvas.heightPx,- video: { frameRate: clock.frameRate, frameCount: exactFrameCount(duration, clock), audio: "silence" },- })) } };- },- },- {- producer: standInProducers.silence,- handler: ({ inputs }) => {- const duration = typedInline<SpeechDuration>(inputs.duration!.value, "stand-in silence duration");- assertSpeechDurationIdentity(duration);- const sampleFrames = Math.round(duration * 48_000);- const request: StandInSilenceRequest = { sampleRate: 48_000, channels: 2, sampleFrames };- verifyStandInSilenceRequest(request);- return { outputs: {}, needs: { audio: canonicalize(request) } };- },- },],} satisfies ComponentPackage;diff --git a/packages/stand-in/src/manifest.ts b/packages/stand-in/src/manifest.tsindex 34a0591c..2c7a946d 100644--- a/packages/stand-in/src/manifest.ts+++ b/packages/stand-in/src/manifest.ts@@ -1,58 +1,36 @@import { artifactDependency, artifactTypes } from "@hypit/artifact";-import { programSpaceDependency, programSpaceTypes } from "@hypit/program-space";import type { CapabilityRef, ModuleManifest, TypeRef } from "@hypit/protocol";import { spatialDependency, spatialTypes } from "@hypit/spatial";-import { speechDependency, speechTypes } from "@hypit/speech";export const standInModuleRef = { name: "@hypit/stand-in", version: "1" } as const;export const standInDependency = { module: standInModuleRef } as const;export const standInTypes = {cardRequest: { module: standInModuleRef, name: "StandInCardRequest" },- silenceRequest: { module: standInModuleRef, name: "StandInSilenceRequest" },} satisfies Record<string, TypeRef>;/** Draw one stand-in card. Served by a local media Provider; it draws, it never generates. */export const standInCapabilities = {drawCard: { module: standInModuleRef, name: "draw-card" },- drawSilence: { module: standInModuleRef, name: "draw-silence" },} satisfies Record<string, CapabilityRef>;export const standInProducers = {- image: { module: standInModuleRef, name: "stand-in-image" },- video: { module: standInModuleRef, name: "stand-in-video" },- silence: { module: standInModuleRef, name: "stand-in-silence" },+ card: { module: standInModuleRef, name: "stand-in-card" },} as const;export const standInManifest: ModuleManifest = {format: "hypit.module@1",name: standInModuleRef.name,version: standInModuleRef.version,- dependencies: [artifactDependency, programSpaceDependency, spatialDependency, speechDependency],+ dependencies: [artifactDependency, spatialDependency],types: Object.values(standInTypes).map((item) => ({ name: item.name })),capabilities: Object.values(standInCapabilities).map((item) => ({ name: item.name, returns: artifactTypes.blob })),producers: [{- name: standInProducers.image.name,+ name: standInProducers.card.name,inputs: [{ name: "canvas", type: spatialTypes.canvas }],outputs: [],needs: [{ name: "image", capability: standInCapabilities.drawCard, returns: artifactTypes.blob }],},- {- name: standInProducers.video.name,- inputs: [- { name: "canvas", type: spatialTypes.canvas },- { name: "duration", type: speechTypes.duration },- { name: "clock", type: programSpaceTypes.clock },- ],- outputs: [],- needs: [{ name: "video", capability: standInCapabilities.drawCard, returns: artifactTypes.blob }],- },- {- name: standInProducers.silence.name,- inputs: [{ name: "duration", type: speechTypes.duration }],- outputs: [],- needs: [{ name: "audio", capability: standInCapabilities.drawSilence, returns: artifactTypes.blob }],- },],};diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 15490c25..a21c0d11 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -2956,9 +2956,9 @@ importers:'@hypit/elaborator':specifier: workspace:*version: link:../elaborator- '@hypit/media':+ '@hypit/media-pipeline':specifier: workspace:*- version: link:../media+ version: link:../media-pipeline'@hypit/program-space':specifier: workspace:*version: link:../program-space