Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(stand-in): the picture a generation would have made, drawn from its draft
kashorin66747d84d86a
diff --git a/package.json b/package.jsonindex 4e091605..20f1ca44 100644--- a/package.json+++ b/package.json@@ -119,6 +119,7 @@"@hypit/semantic-track": "workspace:*","@hypit/semantic-take-estimate": "workspace:*","@hypit/spatial": "workspace:*",+ "@hypit/stand-in": "workspace:*","@hypit/speech": "workspace:*","@hypit/speech-alignment": "workspace:*","@hypit/speech-evidence": "workspace:*",diff --git a/packages/gpt-image/package.json b/packages/gpt-image/package.jsonindex 241a5eed..56af38ef 100644--- a/packages/gpt-image/package.json+++ b/packages/gpt-image/package.json@@ -29,6 +29,7 @@"@hypit/run": "workspace:*","@hypit/seedance": "workspace:*","@hypit/spatial": "workspace:*",- "@hypit/speech": "workspace:*"+ "@hypit/speech": "workspace:*",+ "@hypit/stand-in": "workspace:*"}}diff --git a/packages/gpt-image/src/activation.ts b/packages/gpt-image/src/activation.tsindex 1c2b9921..ed4bf252 100644--- a/packages/gpt-image/src/activation.ts+++ b/packages/gpt-image/src/activation.ts@@ -24,6 +24,7 @@ export const hypitPackage = {components: [gptImageComponent],hostFacets: [gptImageDefinition.hostFacet,+ ...(gptImageDefinition.runFragmentFacet === undefined ? [] : [gptImageDefinition.runFragmentFacet]),createMarkupSurfaceHostFacet({module: gptImageModuleRef,declaration: gptImageMarkupSurfaces.find((item) => item.name === "image")!,diff --git a/packages/gpt-image/test/broll-graph.test.ts b/packages/gpt-image/test/broll-graph.test.tsindex faa08280..ef15695c 100644--- a/packages/gpt-image/test/broll-graph.test.ts+++ b/packages/gpt-image/test/broll-graph.test.ts@@ -41,6 +41,7 @@ import {} from "@hypit/seedance";import { speechManifest } from "@hypit/speech";import { spatialManifest } from "@hypit/spatial";+import { standInManifest } from "@hypit/stand-in";import { svsManifest } from "@hypit/svs";import { textManifest } from "@hypit/text";@@ -67,6 +68,7 @@ function fixture() {svsManifest,speechManifest,seedanceManifest,+ standInManifest,];const closure = createResolvedClosure(manifests);const gpt = gptImageEndpoints.image!;diff --git a/packages/grok-imagine/src/activation.ts b/packages/grok-imagine/src/activation.tsindex dddbd140..20cce3e8 100644--- a/packages/grok-imagine/src/activation.ts+++ b/packages/grok-imagine/src/activation.ts@@ -14,6 +14,7 @@ export const hypitPackage = {components: [grokImagineComponent],hostFacets: [grokImagineDefinition.hostFacet,+ ...(grokImagineDefinition.runFragmentFacet === undefined ? [] : [grokImagineDefinition.runFragmentFacet]),createMarkupSurfaceHostFacet({module: grokImagineModuleRef,declaration: grokImagineMarkupSurfaces.find((item) => item.name === "video")!,diff --git a/packages/media-execution/package.json b/packages/media-execution/package.jsonindex b55fe2c3..3b4f15ca 100644--- a/packages/media-execution/package.json+++ b/packages/media-execution/package.json@@ -12,6 +12,7 @@"@hypit/media-pipeline": "workspace:*","@hypit/program-space": "workspace:*","@hypit/protocol": "workspace:*",- "@hypit/speech": "workspace:*"+ "@hypit/speech": "workspace:*",+ "@hypit/stand-in": "workspace:*"}}diff --git a/packages/media-execution/src/card.ts b/packages/media-execution/src/card.tsnew file mode 100644index 00000000..60b9091f--- /dev/null+++ b/packages/media-execution/src/card.ts@@ -0,0 +1,404 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { deflateSync } from "node:zlib";++import type { MediaStream } from "@hypit/media";+import type { CanonicalValue } from "@hypit/protocol";+import { verifyStandInCardRequest } from "@hypit/stand-in";+import type { StandInCardRequest } from "@hypit/stand-in";++/**+ * A stand-in card, drawn from the request the model would have received.+ *+ * The card is pixels, not a model call: a flat ground coloured by kind, a thick border, a diagonal+ * STAND-IN watermark, the kind in large type, the model, the frame and duration, an excerpt of the+ * prompt, and for video a running timecode and progress bar. Everything is drawn here with a+ * built-in 5x7 bitmap font so the same card comes out of every machine, with or without a font+ * library in its ffmpeg. The PNG is deterministic for a given request.+ */++type Rgb = readonly [number, number, number];++const GROUND: Record<StandInCardRequest["kind"], Rgb> = {+ video: [0x1E, 0x3A, 0x5F],+ image: [0x2E, 0x4A, 0x3A],+};+const ACCENT: Rgb = [0xF5, 0xC5, 0x42];+const TEXT: Rgb = [0xFF, 0xFF, 0xFF];+const MUTED: Rgb = [0xC9, 0xD1, 0xD9];+const INK: Rgb = [0x0B, 0x12, 0x1A];++// A classic 5x7 bitmap face: enough for Latin letters, digits and the punctuation a label carries.+const GLYPHS: Record<string, readonly string[]> = {+ A: [".###.", "#...#", "#...#", "#####", "#...#", "#...#", "#...#"],+ B: ["####.", "#...#", "#...#", "####.", "#...#", "#...#", "####."],+ C: [".###.", "#...#", "#....", "#....", "#....", "#...#", ".###."],+ D: ["####.", "#...#", "#...#", "#...#", "#...#", "#...#", "####."],+ E: ["#####", "#....", "#....", "####.", "#....", "#....", "#####"],+ F: ["#####", "#....", "#....", "####.", "#....", "#....", "#...."],+ G: [".###.", "#...#", "#....", "#.###", "#...#", "#...#", ".####"],+ H: ["#...#", "#...#", "#...#", "#####", "#...#", "#...#", "#...#"],+ I: [".###.", "..#..", "..#..", "..#..", "..#..", "..#..", ".###."],+ J: ["..###", "...#.", "...#.", "...#.", "...#.", "#..#.", ".##.."],+ K: ["#...#", "#..#.", "#.#..", "##...", "#.#..", "#..#.", "#...#"],+ L: ["#....", "#....", "#....", "#....", "#....", "#....", "#####"],+ M: ["#...#", "##.##", "#.#.#", "#.#.#", "#...#", "#...#", "#...#"],+ N: ["#...#", "#...#", "##..#", "#.#.#", "#..##", "#...#", "#...#"],+ O: [".###.", "#...#", "#...#", "#...#", "#...#", "#...#", ".###."],+ P: ["####.", "#...#", "#...#", "####.", "#....", "#....", "#...."],+ Q: [".###.", "#...#", "#...#", "#...#", "#.#.#", "#..#.", ".##.#"],+ R: ["####.", "#...#", "#...#", "####.", "#.#..", "#..#.", "#...#"],+ S: [".####", "#....", "#....", ".###.", "....#", "....#", "####."],+ T: ["#####", "..#..", "..#..", "..#..", "..#..", "..#..", "..#.."],+ U: ["#...#", "#...#", "#...#", "#...#", "#...#", "#...#", ".###."],+ V: ["#...#", "#...#", "#...#", "#...#", "#...#", ".#.#.", "..#.."],+ W: ["#...#", "#...#", "#...#", "#.#.#", "#.#.#", "##.##", "#...#"],+ X: ["#...#", "#...#", ".#.#.", "..#..", ".#.#.", "#...#", "#...#"],+ Y: ["#...#", "#...#", ".#.#.", "..#..", "..#..", "..#..", "..#.."],+ Z: ["#####", "....#", "...#.", "..#..", ".#...", "#....", "#####"],+ "0": [".###.", "#...#", "#..##", "#.#.#", "##..#", "#...#", ".###."],+ "1": ["..#..", ".##..", "..#..", "..#..", "..#..", "..#..", ".###."],+ "2": [".###.", "#...#", "....#", "...#.", "..#..", ".#...", "#####"],+ "3": ["#####", "...#.", "..#..", "...#.", "....#", "#...#", ".###."],+ "4": ["...#.", "..##.", ".#.#.", "#..#.", "#####", "...#.", "...#."],+ "5": ["#####", "#....", "####.", "....#", "....#", "#...#", ".###."],+ "6": ["..##.", ".#...", "#....", "####.", "#...#", "#...#", ".###."],+ "7": ["#####", "....#", "...#.", "..#..", ".#...", ".#...", ".#..."],+ "8": [".###.", "#...#", "#...#", ".###.", "#...#", "#...#", ".###."],+ "9": [".###.", "#...#", "#...#", ".####", "....#", "...#.", ".##.."],+ " ": [".....", ".....", ".....", ".....", ".....", ".....", "....."],+ "-": [".....", ".....", ".....", "#####", ".....", ".....", "....."],+ ":": [".....", "..##.", "..##.", ".....", "..##.", "..##.", "....."],+ ".": [".....", ".....", ".....", ".....", ".....", "..##.", "..##."],+ ",": [".....", ".....", ".....", ".....", "..##.", "..##.", ".#..."],+ "/": ["....#", "....#", "...#.", "..#..", ".#...", "#....", "#...."],+ "(": ["..#..", ".#...", "#....", "#....", "#....", ".#...", "..#.."],+ ")": ["..#..", "...#.", "....#", "....#", "....#", "...#.", "..#.."],+ "'": [".##..", ".##..", ".#...", ".....", ".....", ".....", "....."],+ "?": [".###.", "#...#", "....#", "...#.", "..#..", ".....", "..#.."],+ "!": ["..#..", "..#..", "..#..", "..#..", "..#..", ".....", "..#.."],+ "+": [".....", "..#..", "..#..", "#####", "..#..", "..#..", "....."],+ "_": [".....", ".....", ".....", ".....", ".....", ".....", "#####"],+ "#": [".#.#.", ".#.#.", "#####", ".#.#.", "#####", ".#.#.", ".#.#."],+ "@": [".###.", "#...#", "#.###", "#.#.#", "#.###", "#....", ".###."],+ "&": [".##..", "#..#.", "#..#.", ".##..", "#.#.#", "#..#.", ".##.#"],+ "%": ["##..#", "##..#", "...#.", "..#..", ".#...", "#..##", "#..##"],+ "\"": [".#.#.", ".#.#.", ".#.#.", ".....", ".....", ".....", "....."],+ "⎕": [".###.", ".#.#.", ".#.#.", ".#.#.", ".#.#.", ".#.#.", ".###."],+};+const GLYPH_WIDTH = 5;+const GLYPH_HEIGHT = 7;+const ADVANCE = 6;++function glyph(character: string): readonly string[] {+ const upper = character.toUpperCase();+ return GLYPHS[upper] ?? (upper === "×" ? GLYPHS.X! : GLYPHS["⎕"]!);+}++class Raster {+ readonly width: number;+ readonly height: number;+ readonly pixels: Buffer;++ constructor(width: number, height: number, ground: Rgb) {+ this.width = width;+ this.height = height;+ this.pixels = Buffer.alloc(width * height * 3);+ for (let index = 0; index < width * height; index += 1) {+ this.pixels[index * 3] = ground[0];+ this.pixels[index * 3 + 1] = ground[1];+ this.pixels[index * 3 + 2] = ground[2];+ }+ }++ blend(x: number, y: number, color: Rgb, alpha: number): void {+ if (x < 0 || y < 0 || x >= this.width || y >= this.height) return;+ const offset = (y * this.width + x) * 3;+ for (let channel = 0; channel < 3; channel += 1) {+ const current = this.pixels[offset + channel]!;+ this.pixels[offset + channel] = Math.round(current + (color[channel]! - current) * alpha);+ }+ }++ fill(x: number, y: number, width: number, height: number, color: Rgb, alpha = 1): void {+ const left = Math.max(0, x);+ const top = Math.max(0, y);+ const right = Math.min(this.width, x + width);+ const bottom = Math.min(this.height, y + height);+ for (let row = top; row < bottom; row += 1) {+ for (let column = left; column < right; column += 1) {+ if (alpha >= 1) {+ const offset = (row * this.width + column) * 3;+ this.pixels[offset] = color[0]; this.pixels[offset + 1] = color[1]; this.pixels[offset + 2] = color[2];+ } else {+ this.blend(column, row, color, alpha);+ }+ }+ }+ }++ /** Draw text whose glyph origin is the top-left corner; returns the advance in pixels. */+ text(value: string, x: number, y: number, scale: number, color: Rgb, alpha = 1): number {+ let cursor = x;+ for (const character of value) {+ const rows = glyph(character);+ for (let row = 0; row < GLYPH_HEIGHT; row += 1) {+ for (let column = 0; column < GLYPH_WIDTH; column += 1) {+ if (rows[row]![column] === "#") this.fill(cursor + column * scale, y + row * scale, scale, scale, color, alpha);+ }+ }+ cursor += ADVANCE * scale;+ }+ return cursor - x;+ }++ /** The same text stepped along a 45° line, one glyph per step; the watermark reads diagonally. */+ diagonalText(value: string, x: number, y: number, scale: number, color: Rgb, alpha: number): void {+ const step = GLYPH_WIDTH * scale;+ let index = 0;+ for (const character of value) {+ const rows = glyph(character);+ const originX = x + index * step;+ const originY = y + index * step;+ for (let row = 0; row < GLYPH_HEIGHT; row += 1) {+ for (let column = 0; column < GLYPH_WIDTH; column += 1) {+ if (rows[row]![column] === "#") this.fill(originX + column * scale, originY + row * scale, scale, scale, color, alpha);+ }+ }+ index += 1;+ }+ }++ png(): Uint8Array {+ const stride = this.width * 3;+ const scanlines = Buffer.alloc(this.height * (1 + stride));+ for (let row = 0; row < this.height; row += 1) {+ scanlines[row * (1 + stride)] = 0;+ this.pixels.copy(scanlines, row * (1 + stride) + 1, row * stride, (row + 1) * stride);+ }+ const header = Buffer.alloc(13);+ header.writeUInt32BE(this.width, 0);+ header.writeUInt32BE(this.height, 4);+ header[8] = 8; header[9] = 2; header[10] = 0; header[11] = 0; header[12] = 0;+ return Uint8Array.from(Buffer.concat([+ Buffer.from("\x89PNG\r\n\x1a\n", "binary"),+ Buffer.from(pngChunk("IHDR", header)),+ Buffer.from(pngChunk("IDAT", deflateSync(scanlines))),+ Buffer.from(pngChunk("IEND", new Uint8Array())),+ ]));+ }+}++function crc32(bytes: Uint8Array): number {+ let crc = 0xFFFFFFFF;+ for (const byte of bytes) {+ crc ^= byte;+ for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0);+ }+ return (crc ^ 0xFFFFFFFF) >>> 0;+}++function pngChunk(type: string, data: Uint8Array): Uint8Array {+ const body = new Uint8Array(type.length + data.length);+ for (let index = 0; index < type.length; index += 1) body[index] = type.charCodeAt(index);+ body.set(data, type.length);+ const result = new Uint8Array(12 + data.length);+ const view = new DataView(result.buffer);+ view.setUint32(0, data.length);+ result.set(body, 4);+ view.setUint32(8 + data.length, crc32(body));+ return result;+}++function textWidth(value: string, scale: number): number {+ return Math.max(0, [...value].length * ADVANCE * scale - scale);+}++/** Uppercase, one line per row, at most `maxLines` rows of `maxChars`; the last row ends in an ellipsis. */+function wrap(value: string, maxChars: number, maxLines: number): readonly string[] {+ const words = value.replace(/\s+/gu, " ").trim().split(" ").filter((word) => word.length > 0);+ const lines: string[] = [];+ let current = "";+ for (const word of words) {+ const candidate = current.length === 0 ? word : `${current} ${word}`;+ if (candidate.length <= maxChars) { current = candidate; continue; }+ if (current.length > 0) lines.push(current);+ current = word.length > maxChars ? word.slice(0, maxChars) : word;+ if (lines.length === maxLines) break;+ }+ if (current.length > 0 && lines.length < maxLines) lines.push(current);+ const truncated = lines.join(" ").length < words.join(" ").length;+ if (truncated && lines.length > 0) {+ const last = lines[lines.length - 1]!;+ lines[lines.length - 1] = `${last.slice(0, Math.max(0, maxChars - 3))}...`;+ }+ return lines;+}++function seconds(video: NonNullable<StandInCardRequest["video"]>): number {+ return video.frameCount * video.frameRate.denominator / video.frameRate.numerator;+}++function timecode(frame: number, video: NonNullable<StandInCardRequest["video"]>): string {+ const fps = video.frameRate.numerator / video.frameRate.denominator;+ const total = frame / fps;+ const minutes = Math.floor(total / 60);+ const wholeSeconds = Math.floor(total - minutes * 60);+ const frames = frame - Math.round((minutes * 60 + wholeSeconds) * fps);+ const two = (value: number): string => String(value).padStart(2, "0");+ return `${two(minutes)}:${two(wholeSeconds)}.${two(Math.max(0, frames))}`;+}++type Layout = {+ readonly border: number;+ readonly band: number;+ readonly margin: number;+};++function layout(request: StandInCardRequest): Layout {+ const m = Math.min(request.width, request.height);+ const border = Math.max(6, Math.round(m * 0.03));+ const band = request.kind === "video" ? Math.max(36, Math.round(request.height * 0.06)) : 0;+ return { border, band, margin: border * 2 };+}++/** The still part of the card: everything but the timecode band. */+export function drawStandInCard(request: StandInCardRequest): Uint8Array {+ verifyStandInCardRequest(request);+ const { width, height } = request;+ const m = Math.min(width, height);+ const { border, band, margin } = layout(request);+ const raster = new Raster(width, height, GROUND[request.kind]);++ // Diagonal watermark: stepped 45° lines of STAND-IN across the whole ground, faint.+ const markScale = Math.max(2, Math.floor(m / 220));+ const word = "STAND-IN ";+ const wordSpan = word.length * GLYPH_WIDTH * markScale;+ const lineGap = Math.max(GLYPH_HEIGHT * markScale * 3, Math.round(m * 0.22));+ for (let start = -height; start < width; start += lineGap) {+ for (let along = 0; start + along < width + wordSpan && along < width + height; along += wordSpan) {+ raster.diagonalText(word, start + along, along, markScale, TEXT, 0.09);+ }+ }++ // Border.+ raster.fill(0, 0, width, border, ACCENT);+ raster.fill(0, height - border, width, border, ACCENT);+ raster.fill(0, 0, border, height, ACCENT);+ raster.fill(width - border, 0, border, height, ACCENT);++ // Corner tag.+ const tagScale = Math.max(2, Math.floor(m / 260));+ raster.text("STAND-IN", margin, margin, tagScale, ACCENT);+ const kindTag = request.kind === "video" ? "NOT YET GENERATED" : "NOT YET GENERATED";+ raster.text(kindTag, width - margin - textWidth(kindTag, tagScale), margin, tagScale, MUTED);++ // Centre block: kind, model, frame and duration, prompt excerpt.+ const titleScale = Math.max(4, Math.floor(m / 70));+ const lineScale = Math.max(2, Math.floor(m / 200));+ const promptScale = Math.max(2, Math.floor(m / 240));+ const title = request.kind.toUpperCase();+ const frameLine = request.video === undefined+ ? `${width} X ${height}`+ : `${width} X ${height} ${seconds(request.video).toFixed(2)} S ${(request.video.frameRate.numerator / request.video.frameRate.denominator).toFixed(2)} FPS`;+ const promptLines = wrap(request.prompt, Math.max(8, Math.floor((width - margin * 2) / (ADVANCE * promptScale))), 4);+ const blockHeight = GLYPH_HEIGHT * titleScale + lineScale * 6+ + GLYPH_HEIGHT * lineScale * 2 + lineScale * 4+ + (promptLines.length === 0 ? 0 : lineScale * 8 + promptLines.length * (GLYPH_HEIGHT * promptScale + promptScale * 3));+ let y = Math.round((height - band - blockHeight) / 2);+ const centred = (value: string, scale: number, color: Rgb): void => {+ raster.text(value, Math.round((width - textWidth(value, scale)) / 2), y, scale, color);+ y += GLYPH_HEIGHT * scale;+ };+ centred(title, titleScale, TEXT);+ y += lineScale * 6;+ centred(request.model, lineScale, ACCENT);+ y += lineScale * 4;+ centred(frameLine, lineScale, MUTED);+ if (promptLines.length > 0) {+ y += lineScale * 8;+ for (const line of promptLines) {+ centred(line, promptScale, TEXT);+ y += promptScale * 3;+ }+ }+ return raster.png();+}++/** The band a video card carries under its picture: timecode, frame counter and progress bar. */+export function drawStandInBand(request: StandInCardRequest, frame: number): Uint8Array {+ verifyStandInCardRequest(request);+ const video = request.video;+ if (video === undefined) throw new Error("only a video stand-in has a timecode band");+ const { width } = request;+ const { border, band, margin } = layout(request);+ const raster = new Raster(width, band, INK);+ raster.fill(0, 0, border, band, ACCENT);+ raster.fill(width - border, 0, border, band, ACCENT);+ const scale = Math.max(2, Math.floor((band - border) / (GLYPH_HEIGHT + 4)));+ const textY = Math.round((band - border - GLYPH_HEIGHT * scale) / 2);+ raster.text(timecode(frame, video), margin, textY, scale, TEXT);+ const counter = `F ${frame + 1}/${video.frameCount}`;+ raster.text(counter, width - margin - textWidth(counter, scale), textY, scale, MUTED);+ raster.fill(0, band - border, width, border, INK);+ raster.fill(0, band - border, Math.round(width * (frame + 1) / video.frameCount), border, ACCENT);+ return raster.png();+}++export type StandInDrawingEnvironment = {+ readonly ffmpegPath: string;+ readonly processTimeoutMs: number;+ readonly runFfmpeg: (argv: readonly string[]) => Promise<void>;+ readonly inspectStreams: (path: string) => Promise<readonly MediaStream[]>;+ readonly putBytes: (bytes: Uint8Array, mediaType: string) => Promise<CanonicalValue>;+ readonly putFile: (path: string, mediaType: string) => Promise<CanonicalValue>;+};++export function standInCardNeed(value: CanonicalValue): StandInCardRequest {+ verifyStandInCardRequest(value);+ return value;+}++/** Draw the card; a video card is the still card with its band composited frame by frame by ffmpeg. */+export async function renderStandInCard(+ env: StandInDrawingEnvironment,+ request: StandInCardRequest,+): Promise<CanonicalValue> {+ const still = drawStandInCard(request);+ if (request.video === undefined) return await env.putBytes(still, "image/png");+ const video = request.video;+ const work = await mkdtemp(join(tmpdir(), "hypit-stand-in-"));+ try {+ const base = join(work, "card.png");+ await writeFile(base, still);+ for (let frame = 0; frame < video.frameCount; frame += 1) {+ await writeFile(join(work, `band-${String(frame).padStart(6, "0")}.png`), drawStandInBand(request, frame));+ }+ const fps = `${video.frameRate.numerator}/${video.frameRate.denominator}`;+ const output = join(work, "card.mp4");+ const silence = video.audio === "silence";+ await env.runFfmpeg([+ "-y",+ "-loop", "1", "-framerate", fps, "-i", base,+ "-framerate", fps, "-i", join(work, "band-%06d.png"),+ ...(silence ? ["-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"] : []),+ "-filter_complex", "[0:v][1:v]overlay=0:main_h-overlay_h:shortest=1,format=yuv420p[v]",+ "-map", "[v]", ...(silence ? ["-map", "2:a:0", "-c:a", "aac", "-b:a", "96k", "-shortest"] : []),+ "-frames:v", String(video.frameCount), "-r", fps, "-fps_mode", "cfr",+ "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",+ "-movflags", "+faststart", output,+ ]);+ const streams = await env.inspectStreams(output);+ const visual = streams.find((item) => item.kind === "video");+ const audio = streams.filter((item) => item.kind === "audio");+ if (visual === undefined || visual.kind !== "video" || visual.decodedUnitCount !== video.frameCount+ || visual.width !== request.width || visual.height !== request.height+ || streams.length !== 1 + audio.length || audio.length !== (silence ? 1 : 0)) {+ throw new Error("Stand-in video differs from its requested frame domain");+ }+ return await env.putFile(output, "video/mp4");+ } finally {+ await rm(work, { recursive: true, force: true }).catch(() => {});+ }+}diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.tsindex 267e7204..47d2c97f 100644--- a/packages/media-execution/src/execute.ts+++ b/packages/media-execution/src/execute.ts@@ -32,6 +32,7 @@ import {} from "@hypit/protocol";import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";+import { renderStandInCard, standInCardNeed } from "./card.js";import { parseMediaInspection } from "./probe.js";import {compositeAnimatedWebpFrame,@@ -786,6 +787,41 @@ export async function executeRenderStillVideo(}}+/** Draw a stand-in card for a generated output that has not been generated; a Provider draws, it never generates. */+export async function executeDrawStandInCard(+ env: MediaExecutionEnvironment,+ constraints: CanonicalValue,+): Promise<MediaOperationResult> {+ const request = standInCardNeed(constraints);+ const value = await renderStandInCard({+ ffmpegPath: env.ffmpegPath,+ processTimeoutMs: env.processTimeoutMs,+ runFfmpeg: async (argv) => {+ await runProcess({+ executable: env.ffmpegPath,+ argv,+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ },+ inspectStreams: async (path) => {+ const inspected = await outputInspection({+ path,+ mediaType: "video/mp4",+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ return inspected.streams;+ },+ putBytes: async (bytes, mediaType) => await env.artifacts.put(bytes, mediaType) as unknown as CanonicalValue,+ putFile: async (path, mediaType) => await env.artifacts.putFile(path, mediaType) as unknown as CanonicalValue,+ }, request);+ return artifactResult(value as unknown as BlobRef);+}+type TransformPlan = {readonly videoFilters: readonly string[];readonly audioFilters: readonly string[];diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.tsindex fcc10457..6dbe5f05 100644--- a/packages/media-execution/src/index.ts+++ b/packages/media-execution/src/index.ts@@ -7,8 +7,10 @@ export {executeProjectSpeechEvidenceAudio,executeRenderTimelineAudio,executeRenderStillVideo,+ executeDrawStandInCard,executeTransformMedia,} from "./execute.js";+export { drawStandInBand, drawStandInCard } from "./card.js";export { parseMediaInspection } from "./probe.js";export { verifyCompositableSurfaceBytes } from "./surface.js";export { probeMediaToolchain } from "./toolchain.js";diff --git a/packages/media-track/preview/preview.svml b/packages/media-track/preview/preview.svmlindex 677cf647..3fd79c7e 100644--- a/packages/media-track/preview/preview.svml+++ b/packages/media-track/preview/preview.svml@@ -9,14 +9,18 @@The checked-in Track.png is the package catalogue asset. This Source and its ordinary Runpreserve how it was authored; regenerating the encoded frame is a normal Build.- A placeholder shows the Frame treatment and nothing inside it. To show the component holding a- picture, commit a sample beside this file and satisfy the generated output in `build.svrun`;- the Profile's local media Endpoint still inspects and normalizes it, in Studio as in a Build:+ A placeholder shows the Frame treatment and nothing inside it. To look at the component before+ buying its pictures, stand in for each generation in the Run; the card is derived from the+ model's own draft, and the Profile's local media Endpoint normalizes it, in Studio as in a Build:- <file id="sample-a" type="@hypit/artifact@1#BlobArtifact" from="./sample-a.mp4" media-type="video/mp4"/>- <satisfy output="motion-a.video" candidate="sample-a"/>+ <import from="@hypit/seedance@1" as="seedance"/>+ <fragment id="a-card" using="seedance:mini-stand-in">+ <input name="draft" from="motion-a.draft"/><input name="prompt" from="direction"/><input name="clock" from="clock"/>+ </fragment>+ <satisfy output="motion-a.video" candidate="a-card.video"/>- Candidate selection is explicit in the Run; Studio never replaces missing media itself. -->+ A committed sample works the same way as a `file` Candidate on `motion-a.video`. Candidate+ selection is explicit in the Run; Studio never replaces missing media itself. --><svml><import from="@hypit/script@1"/><import as="wording" from="@hypit/text@1"/>diff --git a/packages/minimax-h3/src/activation.ts b/packages/minimax-h3/src/activation.tsindex 7229333a..e253c792 100644--- a/packages/minimax-h3/src/activation.ts+++ b/packages/minimax-h3/src/activation.ts@@ -4,6 +4,7 @@ import { minimaxH3Component, minimaxH3Definition, minimaxH3Manifest, minimaxH3Moimport { decodeMinimaxFrameVideoSurface, decodeMinimaxReferenceVideoSurface, decodeMinimaxTextVideoSurface } from "./surface.js";export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: minimaxH3Manifest }], components: [minimaxH3Component], hostFacets: [minimaxH3Definition.hostFacet,+ ...(minimaxH3Definition.runFragmentFacet === undefined ? [] : [minimaxH3Definition.runFragmentFacet]),createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,declaration: minimaxH3MarkupSurfaces.find((item) => item.name === "text-video")!, handler: decodeMinimaxTextVideoSurface }),createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,diff --git a/packages/model-kit/package.json b/packages/model-kit/package.jsonindex f1bae183..3132e021 100644--- a/packages/model-kit/package.json+++ b/packages/model-kit/package.json@@ -13,7 +13,10 @@"@hypit/elaborator": "workspace:*","@hypit/generation": "workspace:*","@hypit/host": "workspace:*",+ "@hypit/program-space": "workspace:*","@hypit/protocol": "workspace:*",+ "@hypit/run": "workspace:*",+ "@hypit/stand-in": "workspace:*","@hypit/text": "workspace:*"}}diff --git a/packages/model-kit/src/index.ts b/packages/model-kit/src/index.tsindex 64018f9d..395adff5 100644--- a/packages/model-kit/src/index.ts+++ b/packages/model-kit/src/index.ts@@ -27,7 +27,15 @@ import type {GenerationPortTable,GenerationRequestDraft,} from "@hypit/generation";+import { programSpaceDependency } from "@hypit/program-space";+import type { ProgramClock } from "@hypit/program-space";+import { createRunFragmentHostFacet } from "@hypit/run";+import type { RunFragmentHostFacet } from "@hypit/run";+import { standInDependency } from "@hypit/stand-in";import { textDependency, textTypes } from "@hypit/text";++import { exactModelStandIn, standInCardRequestFromDraft, standInNeed } from "./stand-in.js";+import type { ExactModelStandIn } from "./stand-in.js";import type { Text } from "@hypit/text";import {canonicalize,@@ -87,6 +95,8 @@ export type ExactModelEndpoint = {readonly textBindings: Readonly<Record<string, ExactModelTextBindingEndpoint>>;readonly ports: GenerationPortTable;readonly fragment: ReturnType<typeof sealGraphFragment>;+ /** The drawn stand-in for this model's output; absent for a model whose output is not a picture. */+ readonly standIn?: ExactModelStandIn;};export type ExactModelMediaBindingEndpoint = {@@ -125,6 +135,8 @@ export type ExactModelModule<Key extends string = string> = {};/** Inert declaration of these exact models for Hosts that address one directly. */readonly hostFacet: ExactModelHostFacet;+ /** Run Fragments that stand in for this module's picture outputs, one per model, named `<key>-stand-in`. */+ readonly runFragmentFacet?: RunFragmentHostFacet;};export const exactModelHostAbi = "hypit.exact-model-host@1";@@ -372,7 +384,11 @@ export function defineExactModelModule<const Key extends string>(assert(new Set(keys).size === keys.length, `${options.module.name} repeats an endpoint key`);const models = options.endpoints.map((item) => item.ports.model);assert(new Set(models).size === models.length, `${options.module.name} repeats an exact model`);- const endpointData = options.endpoints.map((spec) => ({ spec, ...endpointRef(options.module, spec) }));+ const endpointData = options.endpoints.map((spec) => {+ const refs = endpointRef(options.module, spec);+ const standIn = exactModelStandIn({ module: options.module, producerName: spec.producerName, draftType: refs.draftType, ports: spec.ports });+ return { spec, ...refs, ...(standIn === undefined ? {} : { standIn }) };+ });const manifest: ModuleManifest = {format: "hypit.module@1",@@ -382,6 +398,8 @@ export function defineExactModelModule<const Key extends string>({ module: generationModuleRef },artifactDependency,textDependency,+ standInDependency,+ programSpaceDependency,],types: endpointData.flatMap((item) => [{@@ -437,6 +455,12 @@ export function defineExactModelModule<const Key extends string>(outputs: [{ name: "request", type: item.requestType }],needs: [],},+ ...(item.standIn === undefined ? [] : [{+ name: item.standIn.producer.name,+ inputs: item.standIn.inputs,+ outputs: [],+ needs: [standInNeed],+ }]),]),};@@ -467,9 +491,13 @@ export function defineExactModelModule<const Key extends string>(textBindings: item.textBindings,ports: item.spec.ports,fragment,+ ...(item.standIn === undefined ? {} : { standIn: item.standIn }),} satisfies ExactModelEndpoint];}));+ const standIns = endpointData.flatMap((item) =>+ item.standIn === undefined ? [] : [[`${item.spec.key}-stand-in`, item.standIn.fragment] as const]);+return {module: { ...options.module },manifest,@@ -551,8 +579,24 @@ export function defineExactModelModule<const Key extends string>(needs: {},}),},+ ...(item.standIn === undefined ? [] : [{+ producer: item.standIn.producer,+ handler: ({ inputs }: ProducerHandlerContext) => {+ const draft = inlineValue<GenerationRequestDraft>(inputs.draft!.value, `${item.spec.key} draft`);+ const prompt = inlineValue<Text>(inputs.prompt!.value, `${item.spec.key} stand-in prompt`);+ const clock = inputs.clock === undefined ? undefined : inlineValue<ProgramClock>(inputs.clock.value, `${item.spec.key} stand-in clock`);+ const request = standInCardRequestFromDraft({ ports: item.spec.ports, draft, prompt, ...(clock === undefined ? {} : { clock }) });+ return { outputs: {}, needs: { card: canonicalize(request as unknown as CanonicalValue) } };+ },+ }]),]),},+ ...(standIns.length === 0 ? {} : {+ runFragmentFacet: createRunFragmentHostFacet({+ name: `${options.module.name}@${options.module.version}`,+ fragments: Object.fromEntries(standIns),+ }),+ }),};}@@ -664,3 +708,5 @@ export function createExactModelPrimaryGenerationFragment(}],});}+export { standInCardRequestFromDraft, standInFrameFromDraft } from "./stand-in.js";+export type { ExactModelStandIn } from "./stand-in.js";diff --git a/packages/model-kit/src/stand-in.ts b/packages/model-kit/src/stand-in.tsnew file mode 100644index 00000000..c7f60585--- /dev/null+++ b/packages/model-kit/src/stand-in.ts@@ -0,0 +1,139 @@+import { artifactTypes } from "@hypit/artifact";+import { sealGraphFragment } from "@hypit/elaborator";+import type { GenerationPortTable, GenerationRequestDraft } from "@hypit/generation";+import { assertProgramClockIdentity, programSpaceTypes } from "@hypit/program-space";+import type { ProgramClock } from "@hypit/program-space";+import type { ModuleRef, ProducerRef, TypeRef } from "@hypit/protocol";+import { sealStandInCardRequest, standInCapabilities } from "@hypit/stand-in";+import type { StandInCardRequest } from "@hypit/stand-in";+import { textTypes } from "@hypit/text";+import type { Text } from "@hypit/text";++/**+ * A stand-in for an exact model's output is derived from that model's own request draft: the+ * frame from its resolution and aspect ports, the duration from its duration port, the model from+ * its port table, the prompt from the same Text edge the model would read. Nothing is restated by+ * hand, so the card follows the Source. What cannot be derived is refused with the reason.+ */++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++/** How many pixels a resolution token names, and on which side. */+const RESOLUTIONS: Readonly<Record<string, { readonly pixels: number; readonly side: "shorter" | "longer" }>> = {+ "480p": { pixels: 480, side: "shorter" },+ "720p": { pixels: 720, side: "shorter" },+ "1080p": { pixels: 1080, side: "shorter" },+ "4k": { pixels: 2160, side: "shorter" },+ "1K": { pixels: 1024, side: "longer" },+ "2K": { pixels: 2048, side: "longer" },+ "4K": { pixels: 4096, side: "longer" },+};++function single(draft: GenerationRequestDraft, port: string): string | number | boolean | undefined {+ const values = draft.ports[port];+ if (values === undefined || values.length === 0) return undefined;+ const value = values[0];+ return typeof value === "object" ? undefined : value;+}++function even(value: number): number {+ const rounded = Math.round(value);+ return rounded % 2 === 0 ? rounded : rounded + 1;+}++export function standInFrameFromDraft(ports: GenerationPortTable, draft: GenerationRequestDraft): { readonly width: number; readonly height: number } {+ const size = single(draft, "size");+ if (typeof size === "string") {+ const match = /^(\d+)\s*[xX×]\s*(\d+)$/u.exec(size.trim());+ assert(match !== null, `${ports.model} size port "${size}" is not WIDTHxHEIGHT; no stand-in frame can be derived`);+ return { width: even(Number(match[1])), height: even(Number(match[2])) };+ }+ const resolution = single(draft, "resolution");+ assert(typeof resolution === "string" && RESOLUTIONS[resolution] !== undefined,+ `${ports.model} names no resolution a stand-in understands (${String(resolution)}); the card cannot know its frame`);+ const aspect = single(draft, "aspectRatio");+ let ratio: { readonly w: number; readonly h: number };+ if (typeof aspect === "string") {+ const match = /^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/u.exec(aspect.trim());+ assert(match !== null, `${ports.model} aspectRatio "${aspect}" is not W:H; no stand-in frame can be derived`);+ ratio = { w: Number(match[1]), h: Number(match[2]) };+ } else {+ assert(ports.result === "image", `${ports.model} declares no aspect ratio; a video stand-in cannot guess its frame`);+ ratio = { w: 1, h: 1 };+ }+ const { pixels, side } = RESOLUTIONS[resolution]!;+ const portrait = ratio.h > ratio.w;+ const fixShorter = side === "shorter";+ // The named pixel count lands on the shorter side (480p, 720p…) or the longer side (1K, 2K…).+ const widthIsFixed = fixShorter ? portrait : !portrait;+ return widthIsFixed+ ? { width: even(pixels), height: even(pixels * ratio.h / ratio.w) }+ : { width: even(pixels * ratio.w / ratio.h), height: even(pixels) };+}++export function standInCardRequestFromDraft(args: {+ readonly ports: GenerationPortTable;+ readonly draft: GenerationRequestDraft;+ readonly prompt: Text;+ readonly clock?: ProgramClock;+}): StandInCardRequest {+ const { ports, draft } = args;+ assert(ports.result !== "audio", `${ports.model} produces audio; a stand-in card is a picture`);+ const frame = standInFrameFromDraft(ports, draft);+ const base = { ...frame, model: ports.model, prompt: args.prompt.value };+ if (ports.result === "image") return sealStandInCardRequest({ kind: "image", ...base });+ const clock = args.clock;+ assert(clock !== undefined, `a video stand-in for ${ports.model} needs the piece's clock`);+ assertProgramClockIdentity(clock);+ const duration = single(draft, "duration");+ assert(typeof duration === "number" && Number.isFinite(duration) && duration > 0,+ `${ports.model} has no fixed duration (${String(duration)}); write the seconds, an automatic duration has no stand-in`);+ const frameCount = Math.round(duration * clock.frameRate.numerator / clock.frameRate.denominator);+ assert(Number.isSafeInteger(frameCount) && frameCount > 0, `${ports.model} duration yields no frames on this clock`);+ const audio = single(draft, "generateAudio") === true;+ return sealStandInCardRequest({ kind: "video", ...base, video: { frameRate: clock.frameRate, frameCount, ...(audio ? { audio: "silence" as const } : {}) } });+}++export type ExactModelStandIn = {+ readonly producer: ProducerRef;+ readonly fragment: ReturnType<typeof sealGraphFragment>;+ /** The export name a Run satisfies with: `image` or `video`, the Blob the model would have made. */+ readonly output: "image" | "video";+ readonly inputs: readonly { readonly name: string; readonly type: TypeRef }[];+};++/** The stand-in shape for one endpoint, or none for a model whose output is not a picture. */+export function exactModelStandIn(args: {+ readonly module: ModuleRef;+ readonly producerName: string;+ readonly draftType: TypeRef;+ readonly ports: GenerationPortTable;+}): ExactModelStandIn | undefined {+ if (args.ports.result === "audio") return undefined;+ const video = args.ports.result === "video";+ const producer = { module: args.module, name: `stand-in-${args.producerName}` };+ const inputs = [+ { name: "draft", type: args.draftType },+ { name: "prompt", type: textTypes.text },+ ...(video ? [{ name: "clock", type: programSpaceTypes.clock }] : []),+ ];+ const fragment = sealGraphFragment({+ inputs,+ operations: [{+ id: "card",+ producer,+ inputs: Object.fromEntries(inputs.map((input) => [input.name, { kind: "fragment-input" as const, name: input.name }])),+ result: { kind: "need", name: "card" },+ }],+ exports: [{+ name: video ? "video" : "image",+ type: artifactTypes.blob,+ root: { kind: "fragment-operation", operation: "card" },+ }],+ });+ return { producer, fragment, output: video ? "video" : "image", inputs };+}++export const standInNeed = { name: "card", capability: standInCapabilities.drawCard, returns: artifactTypes.blob } as const;diff --git a/packages/model-kit/test/stand-in.test.ts b/packages/model-kit/test/stand-in.test.tsnew file mode 100644index 00000000..7242dcff--- /dev/null+++ b/packages/model-kit/test/stand-in.test.ts@@ -0,0 +1,85 @@+import assert from "node:assert/strict";+import test from "node:test";++import { sealGenerationRequestDraft } from "@hypit/generation";+import type { GenerationPortTable } from "@hypit/generation";+import { sealText } from "@hypit/text";++import { defineExactModelModule, standInCardRequestFromDraft, standInFrameFromDraft } from "@hypit/model-kit";++const clip: GenerationPortTable = {+ model: "clip-model",+ result: "video",+ ports: [+ { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },+ { name: "resolution", value: { kind: "enum", values: ["480p", "720p", "1080p"] }, minItems: 1, maxItems: 1 },+ { name: "aspectRatio", value: { kind: "enum", values: ["9:16", "16:9", "1:1"] }, minItems: 1, maxItems: 1 },+ { name: "duration", value: { kind: "enum", values: [-1, 4, 5, 8] }, minItems: 1, maxItems: 1 },+ { name: "generateAudio", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },+ ],+ requires: [],+};+const picture: GenerationPortTable = {+ model: "picture-model",+ result: "image",+ ports: [+ { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },+ { name: "resolution", value: { kind: "enum", values: ["1K", "2K"] }, minItems: 1, maxItems: 1 },+ { name: "aspectRatio", value: { kind: "enum", values: ["3:2", "1:1"] }, minItems: 0, maxItems: 1 },+ ],+ requires: [],+};+const voice: GenerationPortTable = {+ model: "voice-model",+ result: "audio",+ ports: [{ name: "text", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 }],+ requires: [],+};+const clock = { frameRate: { numerator: 30, denominator: 1 } };++test("a stand-in card is derived from the model's draft: frame, duration, model and prompt", () => {+ const draft = sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] });+ const request = standInCardRequestFromDraft({ ports: clip, draft, prompt: sealText("A quiet kitchen"), clock });+ assert.deepEqual(request, {+ kind: "video", width: 720, height: 1280, model: "clip-model", prompt: "A quiet kitchen",+ video: { frameRate: { numerator: 30, denominator: 1 }, frameCount: 150 },+ });+ const spoken = standInCardRequestFromDraft({ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5], generateAudio: [true] }), prompt: sealText("x"), clock });+ assert.equal(spoken.video?.audio, "silence", "a model that would have spoken leaves a silent track to cut against");+ assert.deepEqual(standInFrameFromDraft(clip, sealGenerationRequestDraft(clip, { resolution: ["1080p"], aspectRatio: ["16:9"], duration: [4] })),+ { width: 1920, height: 1080 });+ assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["1K"] })), { width: 1024, height: 1024 });+ assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["2K"], aspectRatio: ["3:2"] })), { width: 2048, height: 1366 });+});++test("what a stand-in cannot derive it refuses with the reason", () => {+ assert.throws(() => standInCardRequestFromDraft({+ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [-1] }),+ prompt: sealText("x"), clock,+ }), /automatic duration has no stand-in/u);+ assert.throws(() => standInCardRequestFromDraft({+ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] }), prompt: sealText("x"),+ }), /needs the piece's clock/u);+ assert.throws(() => standInCardRequestFromDraft({+ ports: voice, draft: sealGenerationRequestDraft(voice, {}), prompt: sealText("x"),+ }), /produces audio/u);+});++test("an exact model module exports one stand-in Run Fragment per picture model and none for audio", () => {+ const module = defineExactModelModule({+ module: { name: "@example/models", version: "1" },+ endpoints: [+ { key: "clip", requestTypeName: "ClipRequest", producerName: "request-clip", ports: clip },+ { key: "picture", requestTypeName: "PictureRequest", producerName: "request-picture", ports: picture },+ { key: "voice", requestTypeName: "VoiceRequest", producerName: "request-voice", ports: voice },+ ],+ });+ assert.deepEqual(Object.keys(module.runFragmentFacet!.implementation.fragments).sort(), ["clip-stand-in", "picture-stand-in"]);+ assert.equal(module.runFragmentFacet!.offers[0], "@example/models@1");+ assert.deepEqual(module.endpoints.clip.standIn!.inputs.map((input) => input.name), ["draft", "prompt", "clock"]);+ assert.deepEqual(module.endpoints.picture.standIn!.inputs.map((input) => input.name), ["draft", "prompt"]);+ assert.equal(module.endpoints.voice.standIn, undefined);+ assert.ok(module.manifest.producers.some((item) => item.name === "stand-in-request-clip"+ && item.needs[0]?.capability.name === "draw-card"));+ assert.ok(module.manifest.dependencies.some((item) => item.module.name === "@hypit/stand-in"));+});diff --git a/packages/nano-banana/src/activation.ts b/packages/nano-banana/src/activation.tsindex ff2c6ea4..f8bd921c 100644--- a/packages/nano-banana/src/activation.ts+++ b/packages/nano-banana/src/activation.ts@@ -6,6 +6,7 @@ import {import { decodeNanoBananaImageSurface, decodeNanoBananaProImageSurface } from "./surface.js";export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: nanoBananaManifest }], components: [nanoBananaComponent], hostFacets: [nanoBananaDefinition.hostFacet,+ ...(nanoBananaDefinition.runFragmentFacet === undefined ? [] : [nanoBananaDefinition.runFragmentFacet]),createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,declaration: nanoBananaMarkupSurfaces.find((item) => item.name === "image")!, handler: decodeNanoBananaImageSurface }),createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.jsonindex a7d6fe7f..755c9d2c 100644--- a/packages/provider-media-local/package.json+++ b/packages/provider-media-local/package.json@@ -20,7 +20,8 @@"@hypit/runtime": "workspace:*","@hypit/runtime-kit": "workspace:*","@hypit/runtime-host-node": "workspace:*",- "@hypit/speech": "workspace:*"+ "@hypit/speech": "workspace:*",+ "@hypit/stand-in": "workspace:*"},"devDependencies": {"@hypit/driver-node": "workspace:*"diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.tsindex f85e8b57..b231a521 100644--- a/packages/provider-media-local/src/provider.ts+++ b/packages/provider-media-local/src/provider.ts@@ -9,10 +9,12 @@ import {executeProjectSpeechEvidenceAudio,executeRenderTimelineAudio,executeRenderStillVideo,+ executeDrawStandInCard,executeTransformMedia,} from "@hypit/media-execution";import type { MediaExecutionEnvironment, MediaOperationResult } from "@hypit/media-execution";import { mediaPipelineCapabilities } from "@hypit/media-pipeline";+import { standInCapabilities } from "@hypit/stand-in";import { isStreamingResourceStore } from "@hypit/runtime";import { speechTypes } from "@hypit/speech";import { defineEndpointPackage } from "@hypit/endpoint-kit";@@ -119,6 +121,12 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptionsreturns: artifactTypes.blob,handler: operation(executeRenderStillVideo),},+ {+ lifecycle: "immediate" as const,+ capability: standInCapabilities.drawCard,+ returns: artifactTypes.blob,+ handler: operation(executeDrawStandInCard),+ },{lifecycle: "immediate" as const,capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.tsindex 0f1bf96b..bbf87a17 100644--- a/packages/provider-media-local/test/provider.test.ts+++ b/packages/provider-media-local/test/provider.test.ts@@ -22,6 +22,7 @@ import {selectMediaStreams,} from "@hypit/media-pipeline";import { canonicalize } from "@hypit/protocol";+import { standInCapabilities } from "@hypit/stand-in";import type { CapabilityRef, CanonicalValue, Need, TypeRef } from "@hypit/protocol";import { createLocalMediaProvider } from "../src/index.js";@@ -827,3 +828,57 @@ test("local media Provider executes an end-aligned loop from the exact authoredawait rm(root, { recursive: true, force: true });}});++function pngSize(bytes: Uint8Array): { readonly width: number; readonly height: number } {+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);+ assert.equal(Buffer.from(bytes.subarray(1, 4)).toString("ascii"), "PNG");+ return { width: view.getUint32(16), height: view.getUint32(20) };+}++test("local media Provider draws a stand-in card as a picture and as an exact silent clip", {+ skip: !hasMediaBinaries && "ffmpeg is not installed",+}, async () => {+ const resources = new MemoryResourceStore();+ const fulfillBlob = async (request: Need): Promise<Awaited<ReturnType<MemoryResourceStore["put"]>>> => {+ const provider = await handlerFor(request);+ const result = await provider.handler({+ command: { kind: "fulfill-need", id: `command:${request.id}`, need: request },+ need: request,+ resources,+ credentials: {},+ });+ assert.equal(result.value.kind, "blob");+ return result.value as Awaited<ReturnType<MemoryResourceStore["put"]>>;+ };+ const picture = await fulfillBlob(need("need:card-image", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({+ kind: "image", width: 640, height: 400, model: "picture-model", prompt: "A quiet kitchen counter with fresh basil",+ })));+ assert.equal(picture.kind, "blob");+ assert.equal(picture.mediaType, "image/png");+ const bytes = await resources.get(picture.resource);+ assert.ok(bytes !== undefined);+ assert.deepEqual(pngSize(bytes), { width: 640, height: 400 });++ const clip = await fulfillBlob(need("need:card-video", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({+ kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Locked medium close-up",+ video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12 },+ })));+ assert.equal(clip.mediaType, "video/mp4");+ const inspection = await inspectArtifact(resources, clip);+ const visual = inspection.streams.find((item) => item.kind === "video");+ assert.ok(visual !== undefined && visual.kind === "video");+ assert.equal(inspection.streams.length, 1, "a stand-in clip carries no sound unless the model would have");+ assert.equal(visual.width, 320);+ assert.equal(visual.height, 568);+ assert.equal(visual.decodedUnitCount, 12);++ const spoken = await fulfillBlob(need("need:card-spoken", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({+ kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Spoken dialogue",+ video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12, audio: "silence" },+ })));+ const spokenStreams = (await inspectArtifact(resources, spoken)).streams;+ const sound = spokenStreams.find((item) => item.kind === "audio");+ assert.ok(sound !== undefined && sound.kind === "audio", "a model that would have spoken leaves a silent track");+ assert.equal(sound.sampleRate, 48_000);+ assert.equal(spokenStreams.length, 2);+});diff --git a/packages/seedance/src/activation.ts b/packages/seedance/src/activation.tsindex c4bda476..912cb520 100644--- a/packages/seedance/src/activation.ts+++ b/packages/seedance/src/activation.ts@@ -17,6 +17,7 @@ export const hypitPackage = {components: [seedanceComponent],hostFacets: [seedanceDefinition.hostFacet,+ ...(seedanceDefinition.runFragmentFacet === undefined ? [] : [seedanceDefinition.runFragmentFacet]),createMarkupSurfaceHostFacet({module: seedanceModuleRef,declaration: seedanceMarkupSurfaces.find((item) => item.name === "text-video")!,diff --git a/packages/seedream/src/activation.ts b/packages/seedream/src/activation.tsindex 2ccbedad..1bccf402 100644--- a/packages/seedream/src/activation.ts+++ b/packages/seedream/src/activation.ts@@ -4,6 +4,7 @@ import { seedreamComponent, seedreamDefinition, seedreamManifest, seedreamModuleimport { decodeSeedreamReferenceImageSurface, decodeSeedreamTextImageSurface } from "./surface.js";export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: seedreamManifest }], components: [seedreamComponent], hostFacets: [seedreamDefinition.hostFacet,+ ...(seedreamDefinition.runFragmentFacet === undefined ? [] : [seedreamDefinition.runFragmentFacet]),createMarkupSurfaceHostFacet({ module: seedreamModuleRef,declaration: seedreamMarkupSurfaces.find((item) => item.name === "text-image")!, handler: decodeSeedreamTextImageSurface }),createMarkupSurfaceHostFacet({ module: seedreamModuleRef,diff --git a/packages/stand-in/README.md b/packages/stand-in/README.mdnew file mode 100644index 00000000..d06f5082--- /dev/null+++ b/packages/stand-in/README.md@@ -0,0 +1,29 @@+# `@hypit/stand-in`++A stand-in is what a generated output looks like before it is generated: a designed card, drawn+instead of bought, that the author selects as a Candidate for that output in an ordinary Run. The+card is derived from the model's own request — kind, frame, model, duration, prompt — so it follows+the Source when the Source changes, and it goes through the same inspect and normalize steps as the+real output would.++This module owns the vocabulary only: the `StandInCardRequest` type and the `draw-card` capability.+Every exact-model package built with `@hypit/model-kit` exports one Run Fragment per model that+turns its draft into that request; a local media Provider draws the card (`@hypit/media-execution`).+Nothing here is a mock: the Provider that serves `draw-card` really draws a card, prices it as+local work, and never pretends to be the model.++```xml+<import from="@hypit/seedance@1" as="seedance"/>+<fragment id="kitchen-card" using="seedance:mini-stand-in">+ <input name="draft" from="kitchen.draft"/>+ <input name="prompt" from="kitchen-direction"/>+ <input name="clock" from="clock"/>+</fragment>+<satisfy output="kitchen.video" candidate="kitchen-card.video"/>+```++Remove the `satisfy` line and build again to buy the real output. The card carries a diagonal+STAND-IN watermark, its kind in large type, the model, the frame and duration, an excerpt of the+prompt, and — for video — a running timecode and progress bar, so a cut that lands on the wrong+frame is visible in the preview. Text on the card is drawn with a built-in bitmap font: Latin+letters, digits and common punctuation; other characters appear as boxes.diff --git a/packages/stand-in/package.json b/packages/stand-in/package.jsonnew file mode 100644index 00000000..2966fa97--- /dev/null+++ b/packages/stand-in/package.json@@ -0,0 +1,19 @@+{+ "name": "@hypit/stand-in",+ "version": "0.0.0-dev",+ "license": "SEE LICENSE IN LICENSE",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "hypit": {+ "activation": "./src/activation.ts"+ },+ "dependencies": {+ "@hypit/artifact": "workspace:*",+ "@hypit/component-kit": "workspace:*",+ "@hypit/media": "workspace:*",+ "@hypit/protocol": "workspace:*"+ }+}diff --git a/packages/stand-in/src/activation.ts b/packages/stand-in/src/activation.tsnew file mode 100644index 00000000..d38455b1--- /dev/null+++ b/packages/stand-in/src/activation.ts@@ -0,0 +1,10 @@+import { standInComponent, standInManifest } from "./index.js";++export const hypitPackage = {+ format: "hypit.node-package@1" as const,+ modules: [{ manifest: standInManifest }],+ components: [standInComponent],+ hostFacets: [],+};++export default hypitPackage;diff --git a/packages/stand-in/src/index.ts b/packages/stand-in/src/index.tsnew file mode 100644index 00000000..0be29e75--- /dev/null+++ b/packages/stand-in/src/index.ts@@ -0,0 +1,87 @@+import type { ComponentPackage } from "@hypit/component-kit";+import type { MediaRational } from "@hypit/media";+import { canonicalize } from "@hypit/protocol";+import type { CanonicalValue, StoredValue } from "@hypit/protocol";++import { standInTypes } from "./manifest.js";++export {+ standInCapabilities,+ standInDependency,+ standInManifest,+ standInModuleRef,+ standInTypes,+} from "./manifest.js";++/**+ * What a stand-in card says about the output it stands in for. Every field is derived from the+ * declaration the card replaces — the model's own request — never restated by hand, so the card+ * follows the Source when the Source changes.+ */+export type StandInCardRequest = {+ readonly kind: "image" | "video";+ /** Even, positive pixel dimensions of the card, the frame the real output would have. */+ readonly width: number;+ readonly height: number;+ /** The exact model that would have generated the output. */+ readonly model: string;+ /** The prompt the model would have received; the card prints an excerpt. */+ readonly prompt: string;+ /** A video card runs for exactly this many frames at this clock and carries a running timecode. */+ readonly video?: {+ readonly frameRate: MediaRational;+ readonly frameCount: number;+ /** The model would have generated sound, so the card carries a silent 48 kHz stereo track. */+ readonly audio?: "silence";+ };+};++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++function evenPositive(value: unknown, subject: string): number {+ assert(typeof value === "number" && Number.isSafeInteger(value) && value > 0 && value % 2 === 0,+ `${subject} must be a positive even integer`);+ return value;+}++export function verifyStandInCardRequest(value: unknown): asserts value is StandInCardRequest {+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "StandInCardRequest must be an object");+ const item = value as Record<string, unknown>;+ assert(item.kind === "image" || item.kind === "video", "StandInCardRequest kind must be image or video");+ evenPositive(item.width, "StandInCardRequest width");+ evenPositive(item.height, "StandInCardRequest height");+ assert(typeof item.model === "string" && item.model.trim().length > 0, "StandInCardRequest model must name the model");+ assert(typeof item.prompt === "string", "StandInCardRequest prompt must be a string");+ if (item.kind === "video") {+ const video = item.video as Record<string, unknown> | undefined;+ assert(video !== undefined && typeof video === "object", "a video StandInCardRequest needs its frame domain");+ const rate = video.frameRate as Record<string, unknown> | undefined;+ assert(rate !== undefined && Number.isSafeInteger(rate.numerator) && (rate.numerator as number) > 0+ && Number.isSafeInteger(rate.denominator) && (rate.denominator as number) > 0,+ "StandInCardRequest frameRate must be a positive rational");+ assert(Number.isSafeInteger(video.frameCount) && (video.frameCount as number) > 0,+ "StandInCardRequest frameCount must be a positive integer");+ assert(video.audio === undefined || video.audio === "silence", "StandInCardRequest audio can only be silence");+ } else {+ assert(item.video === undefined, "an image StandInCardRequest has no frame domain");+ }+}++export function sealStandInCardRequest(value: StandInCardRequest): StandInCardRequest {+ verifyStandInCardRequest(value);+ return canonicalize(value as unknown as CanonicalValue) as unknown as StandInCardRequest;+}++function inline(value: StoredValue, subject: string): unknown {+ assert(value.kind === "inline", `${subject} must be inline`);+ return value.value;+}++export const standInComponent = {+ validators: [{+ type: standInTypes.cardRequest,+ handler: ({ value }) => { verifyStandInCardRequest(inline(value, "StandInCardRequest")); },+ }],+} satisfies ComponentPackage;diff --git a/packages/stand-in/src/manifest.ts b/packages/stand-in/src/manifest.tsnew file mode 100644index 00000000..0a1f5b18--- /dev/null+++ b/packages/stand-in/src/manifest.ts@@ -0,0 +1,24 @@+import { artifactDependency, artifactTypes } from "@hypit/artifact";+import type { CapabilityRef, ModuleManifest, TypeRef } from "@hypit/protocol";++export const standInModuleRef = { name: "@hypit/stand-in", version: "1" } as const;+export const standInDependency = { module: standInModuleRef } as const;++export const standInTypes = {+ cardRequest: { module: standInModuleRef, name: "StandInCardRequest" },+} satisfies Record<string, TypeRef>;++/** Draw one stand-in card. Served by a local media Provider; it draws, it never generates. */+export const standInCapabilities = {+ drawCard: { module: standInModuleRef, name: "draw-card" },+} satisfies Record<string, CapabilityRef>;++export const standInManifest: ModuleManifest = {+ format: "hypit.module@1",+ name: standInModuleRef.name,+ version: standInModuleRef.version,+ dependencies: [artifactDependency],+ types: Object.values(standInTypes).map((item) => ({ name: item.name })),+ capabilities: Object.values(standInCapabilities).map((item) => ({ name: item.name, returns: artifactTypes.blob })),+ producers: [],+};diff --git a/packages/studio/README.md b/packages/studio/README.mdindex 04be864c..9ef25f75 100644--- a/packages/studio/README.md+++ b/packages/studio/README.md@@ -83,7 +83,9 @@ that declare `local` pricing, such as the FFmpeg media Endpoint that inspects annormalizes a media file). Resolution is the same as `plan` and a Build use,including the Profile's bindings, minus every priced Endpoint; a Need that reachesa priced Endpoint or none at all stops Studio with the capability named, and the-author supplies a Candidate for that output or builds the Run.+author supplies a Candidate for that output or builds the Run. A stand-in card+(`@hypit/stand-in`, one `<model>-stand-in` Run Fragment per exact model) is such a+local Need: the picture a generation would have made, drawn from its draft instead.Studio and an encoded review use the same ordinary Run. Studio evaluates itsfree display closure in the browser; building that Run evaluates thediff --git a/packages/video-cli/test/packages.ts b/packages/video-cli/test/packages.tsindex e6eb38a6..0086c6be 100644--- a/packages/video-cli/test/packages.ts+++ b/packages/video-cli/test/packages.ts@@ -23,6 +23,7 @@ import speech from "../../speech/src/activation.js";import speechEvidence from "../../speech-evidence/src/activation.js";import semanticTrack from "../../semantic-track/src/activation.js";import spatial from "../../spatial/src/activation.js";+import standIn from "../../stand-in/src/activation.js";import temporal from "../../temporal/src/activation.js";import visualIr from "../../visual-ir/src/activation.js";import svs from "../../svs/src/activation.js";@@ -59,6 +60,7 @@ export const videoTestPackages = [bind("@hypit/speech-track", speechTrack),bind("@hypit/whisperx", whisperX),bind("@hypit/spatial", spatial),+ bind("@hypit/stand-in", standIn),bind("@hypit/temporal", temporal),bind("@hypit/media-track", mediaTrack),bind("@hypit/typography-track", typographyTrack),diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 9db6e5e1..82d13c72 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -234,6 +234,9 @@ importers:'@hypit/speech-track-studio':specifier: workspace:*version: link:packages/speech-track-studio+ '@hypit/stand-in':+ specifier: workspace:*+ version: link:packages/stand-in'@hypit/store-sqlite':specifier: workspace:*version: link:packages/store-sqlite@@ -1456,6 +1459,9 @@ importers:'@hypit/speech':specifier: workspace:*version: link:../speech+ '@hypit/stand-in':+ specifier: workspace:*+ version: link:../stand-inpackages/grok-imagine:dependencies:@@ -1672,6 +1678,9 @@ importers:'@hypit/speech':specifier: workspace:*version: link:../speech+ '@hypit/stand-in':+ specifier: workspace:*+ version: link:../stand-inpackages/media-pipeline:dependencies:@@ -1828,9 +1837,18 @@ importers:'@hypit/host':specifier: workspace:*version: link:../host+ '@hypit/program-space':+ specifier: workspace:*+ version: link:../program-space'@hypit/protocol':specifier: workspace:*version: link:../protocol+ '@hypit/run':+ specifier: workspace:*+ version: link:../run+ '@hypit/stand-in':+ specifier: workspace:*+ version: link:../stand-in'@hypit/text':specifier: workspace:*version: link:../text@@ -2168,6 +2186,9 @@ importers:'@hypit/speech':specifier: workspace:*version: link:../speech+ '@hypit/stand-in':+ specifier: workspace:*+ version: link:../stand-indevDependencies:'@hypit/driver-node':specifier: workspace:*@@ -2915,6 +2936,21 @@ importers:specifier: workspace:*version: link:../studio-adapter+ packages/stand-in:+ dependencies:+ '@hypit/artifact':+ specifier: workspace:*+ version: link:../artifact+ '@hypit/component-kit':+ specifier: workspace:*+ version: link:../component-kit+ '@hypit/media':+ specifier: workspace:*+ version: link:../media+ '@hypit/protocol':+ specifier: workspace:*+ version: link:../protocol+packages/store-sqlite:dependencies:'@hypit/core':