Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat(stand-in): the picture a generation would have made, drawn from its draft

kashorin66747d84d86a
diff --git a/package.json b/package.json
index 4e091605..20f1ca44 100644
--- a/package.json
+++ b/package.json
@@ -119,6 +119,7 @@
"@hypit/semantic-track": "workspace:*",
"@hypit/semantic-take-estimate": "workspace:*",
"@hypit/spatial": "workspace:*",
+ "@hypit/stand-in": "workspace:*",
"@hypit/speech": "workspace:*",
"@hypit/speech-alignment": "workspace:*",
"@hypit/speech-evidence": "workspace:*",
diff --git a/packages/gpt-image/package.json b/packages/gpt-image/package.json
index 241a5eed..56af38ef 100644
--- a/packages/gpt-image/package.json
+++ b/packages/gpt-image/package.json
@@ -29,6 +29,7 @@
"@hypit/run": "workspace:*",
"@hypit/seedance": "workspace:*",
"@hypit/spatial": "workspace:*",
- "@hypit/speech": "workspace:*"
+ "@hypit/speech": "workspace:*",
+ "@hypit/stand-in": "workspace:*"
}
}
diff --git a/packages/gpt-image/src/activation.ts b/packages/gpt-image/src/activation.ts
index 1c2b9921..ed4bf252 100644
--- a/packages/gpt-image/src/activation.ts
+++ b/packages/gpt-image/src/activation.ts
@@ -24,6 +24,7 @@ export const hypitPackage = {
components: [gptImageComponent],
hostFacets: [
gptImageDefinition.hostFacet,
+ ...(gptImageDefinition.runFragmentFacet === undefined ? [] : [gptImageDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: gptImageModuleRef,
declaration: gptImageMarkupSurfaces.find((item) => item.name === "image")!,
diff --git a/packages/gpt-image/test/broll-graph.test.ts b/packages/gpt-image/test/broll-graph.test.ts
index faa08280..ef15695c 100644
--- a/packages/gpt-image/test/broll-graph.test.ts
+++ b/packages/gpt-image/test/broll-graph.test.ts
@@ -41,6 +41,7 @@ import {
} from "@hypit/seedance";
import { speechManifest } from "@hypit/speech";
import { spatialManifest } from "@hypit/spatial";
+import { standInManifest } from "@hypit/stand-in";
import { svsManifest } from "@hypit/svs";
import { textManifest } from "@hypit/text";
@@ -67,6 +68,7 @@ function fixture() {
svsManifest,
speechManifest,
seedanceManifest,
+ standInManifest,
];
const closure = createResolvedClosure(manifests);
const gpt = gptImageEndpoints.image!;
diff --git a/packages/grok-imagine/src/activation.ts b/packages/grok-imagine/src/activation.ts
index dddbd140..20cce3e8 100644
--- a/packages/grok-imagine/src/activation.ts
+++ b/packages/grok-imagine/src/activation.ts
@@ -14,6 +14,7 @@ export const hypitPackage = {
components: [grokImagineComponent],
hostFacets: [
grokImagineDefinition.hostFacet,
+ ...(grokImagineDefinition.runFragmentFacet === undefined ? [] : [grokImagineDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: grokImagineModuleRef,
declaration: grokImagineMarkupSurfaces.find((item) => item.name === "video")!,
diff --git a/packages/media-execution/package.json b/packages/media-execution/package.json
index b55fe2c3..3b4f15ca 100644
--- a/packages/media-execution/package.json
+++ b/packages/media-execution/package.json
@@ -12,6 +12,7 @@
"@hypit/media-pipeline": "workspace:*",
"@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
- "@hypit/speech": "workspace:*"
+ "@hypit/speech": "workspace:*",
+ "@hypit/stand-in": "workspace:*"
}
}
diff --git a/packages/media-execution/src/card.ts b/packages/media-execution/src/card.ts
new file mode 100644
index 00000000..60b9091f
--- /dev/null
+++ b/packages/media-execution/src/card.ts
@@ -0,0 +1,404 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { deflateSync } from "node:zlib";
+
+import type { MediaStream } from "@hypit/media";
+import type { CanonicalValue } from "@hypit/protocol";
+import { verifyStandInCardRequest } from "@hypit/stand-in";
+import type { StandInCardRequest } from "@hypit/stand-in";
+
+/**
+ * A stand-in card, drawn from the request the model would have received.
+ *
+ * The card is pixels, not a model call: a flat ground coloured by kind, a thick border, a diagonal
+ * STAND-IN watermark, the kind in large type, the model, the frame and duration, an excerpt of the
+ * prompt, and for video a running timecode and progress bar. Everything is drawn here with a
+ * built-in 5x7 bitmap font so the same card comes out of every machine, with or without a font
+ * library in its ffmpeg. The PNG is deterministic for a given request.
+ */
+
+type Rgb = readonly [number, number, number];
+
+const GROUND: Record<StandInCardRequest["kind"], Rgb> = {
+ video: [0x1E, 0x3A, 0x5F],
+ image: [0x2E, 0x4A, 0x3A],
+};
+const ACCENT: Rgb = [0xF5, 0xC5, 0x42];
+const TEXT: Rgb = [0xFF, 0xFF, 0xFF];
+const MUTED: Rgb = [0xC9, 0xD1, 0xD9];
+const INK: Rgb = [0x0B, 0x12, 0x1A];
+
+// A classic 5x7 bitmap face: enough for Latin letters, digits and the punctuation a label carries.
+const GLYPHS: Record<string, readonly string[]> = {
+ A: [".###.", "#...#", "#...#", "#####", "#...#", "#...#", "#...#"],
+ B: ["####.", "#...#", "#...#", "####.", "#...#", "#...#", "####."],
+ C: [".###.", "#...#", "#....", "#....", "#....", "#...#", ".###."],
+ D: ["####.", "#...#", "#...#", "#...#", "#...#", "#...#", "####."],
+ E: ["#####", "#....", "#....", "####.", "#....", "#....", "#####"],
+ F: ["#####", "#....", "#....", "####.", "#....", "#....", "#...."],
+ G: [".###.", "#...#", "#....", "#.###", "#...#", "#...#", ".####"],
+ H: ["#...#", "#...#", "#...#", "#####", "#...#", "#...#", "#...#"],
+ I: [".###.", "..#..", "..#..", "..#..", "..#..", "..#..", ".###."],
+ J: ["..###", "...#.", "...#.", "...#.", "...#.", "#..#.", ".##.."],
+ K: ["#...#", "#..#.", "#.#..", "##...", "#.#..", "#..#.", "#...#"],
+ L: ["#....", "#....", "#....", "#....", "#....", "#....", "#####"],
+ M: ["#...#", "##.##", "#.#.#", "#.#.#", "#...#", "#...#", "#...#"],
+ N: ["#...#", "#...#", "##..#", "#.#.#", "#..##", "#...#", "#...#"],
+ O: [".###.", "#...#", "#...#", "#...#", "#...#", "#...#", ".###."],
+ P: ["####.", "#...#", "#...#", "####.", "#....", "#....", "#...."],
+ Q: [".###.", "#...#", "#...#", "#...#", "#.#.#", "#..#.", ".##.#"],
+ R: ["####.", "#...#", "#...#", "####.", "#.#..", "#..#.", "#...#"],
+ S: [".####", "#....", "#....", ".###.", "....#", "....#", "####."],
+ T: ["#####", "..#..", "..#..", "..#..", "..#..", "..#..", "..#.."],
+ U: ["#...#", "#...#", "#...#", "#...#", "#...#", "#...#", ".###."],
+ V: ["#...#", "#...#", "#...#", "#...#", "#...#", ".#.#.", "..#.."],
+ W: ["#...#", "#...#", "#...#", "#.#.#", "#.#.#", "##.##", "#...#"],
+ X: ["#...#", "#...#", ".#.#.", "..#..", ".#.#.", "#...#", "#...#"],
+ Y: ["#...#", "#...#", ".#.#.", "..#..", "..#..", "..#..", "..#.."],
+ Z: ["#####", "....#", "...#.", "..#..", ".#...", "#....", "#####"],
+ "0": [".###.", "#...#", "#..##", "#.#.#", "##..#", "#...#", ".###."],
+ "1": ["..#..", ".##..", "..#..", "..#..", "..#..", "..#..", ".###."],
+ "2": [".###.", "#...#", "....#", "...#.", "..#..", ".#...", "#####"],
+ "3": ["#####", "...#.", "..#..", "...#.", "....#", "#...#", ".###."],
+ "4": ["...#.", "..##.", ".#.#.", "#..#.", "#####", "...#.", "...#."],
+ "5": ["#####", "#....", "####.", "....#", "....#", "#...#", ".###."],
+ "6": ["..##.", ".#...", "#....", "####.", "#...#", "#...#", ".###."],
+ "7": ["#####", "....#", "...#.", "..#..", ".#...", ".#...", ".#..."],
+ "8": [".###.", "#...#", "#...#", ".###.", "#...#", "#...#", ".###."],
+ "9": [".###.", "#...#", "#...#", ".####", "....#", "...#.", ".##.."],
+ " ": [".....", ".....", ".....", ".....", ".....", ".....", "....."],
+ "-": [".....", ".....", ".....", "#####", ".....", ".....", "....."],
+ ":": [".....", "..##.", "..##.", ".....", "..##.", "..##.", "....."],
+ ".": [".....", ".....", ".....", ".....", ".....", "..##.", "..##."],
+ ",": [".....", ".....", ".....", ".....", "..##.", "..##.", ".#..."],
+ "/": ["....#", "....#", "...#.", "..#..", ".#...", "#....", "#...."],
+ "(": ["..#..", ".#...", "#....", "#....", "#....", ".#...", "..#.."],
+ ")": ["..#..", "...#.", "....#", "....#", "....#", "...#.", "..#.."],
+ "'": [".##..", ".##..", ".#...", ".....", ".....", ".....", "....."],
+ "?": [".###.", "#...#", "....#", "...#.", "..#..", ".....", "..#.."],
+ "!": ["..#..", "..#..", "..#..", "..#..", "..#..", ".....", "..#.."],
+ "+": [".....", "..#..", "..#..", "#####", "..#..", "..#..", "....."],
+ "_": [".....", ".....", ".....", ".....", ".....", ".....", "#####"],
+ "#": [".#.#.", ".#.#.", "#####", ".#.#.", "#####", ".#.#.", ".#.#."],
+ "@": [".###.", "#...#", "#.###", "#.#.#", "#.###", "#....", ".###."],
+ "&": [".##..", "#..#.", "#..#.", ".##..", "#.#.#", "#..#.", ".##.#"],
+ "%": ["##..#", "##..#", "...#.", "..#..", ".#...", "#..##", "#..##"],
+ "\"": [".#.#.", ".#.#.", ".#.#.", ".....", ".....", ".....", "....."],
+ "⎕": [".###.", ".#.#.", ".#.#.", ".#.#.", ".#.#.", ".#.#.", ".###."],
+};
+const GLYPH_WIDTH = 5;
+const GLYPH_HEIGHT = 7;
+const ADVANCE = 6;
+
+function glyph(character: string): readonly string[] {
+ const upper = character.toUpperCase();
+ return GLYPHS[upper] ?? (upper === "×" ? GLYPHS.X! : GLYPHS["⎕"]!);
+}
+
+class Raster {
+ readonly width: number;
+ readonly height: number;
+ readonly pixels: Buffer;
+
+ constructor(width: number, height: number, ground: Rgb) {
+ this.width = width;
+ this.height = height;
+ this.pixels = Buffer.alloc(width * height * 3);
+ for (let index = 0; index < width * height; index += 1) {
+ this.pixels[index * 3] = ground[0];
+ this.pixels[index * 3 + 1] = ground[1];
+ this.pixels[index * 3 + 2] = ground[2];
+ }
+ }
+
+ blend(x: number, y: number, color: Rgb, alpha: number): void {
+ if (x < 0 || y < 0 || x >= this.width || y >= this.height) return;
+ const offset = (y * this.width + x) * 3;
+ for (let channel = 0; channel < 3; channel += 1) {
+ const current = this.pixels[offset + channel]!;
+ this.pixels[offset + channel] = Math.round(current + (color[channel]! - current) * alpha);
+ }
+ }
+
+ fill(x: number, y: number, width: number, height: number, color: Rgb, alpha = 1): void {
+ const left = Math.max(0, x);
+ const top = Math.max(0, y);
+ const right = Math.min(this.width, x + width);
+ const bottom = Math.min(this.height, y + height);
+ for (let row = top; row < bottom; row += 1) {
+ for (let column = left; column < right; column += 1) {
+ if (alpha >= 1) {
+ const offset = (row * this.width + column) * 3;
+ this.pixels[offset] = color[0]; this.pixels[offset + 1] = color[1]; this.pixels[offset + 2] = color[2];
+ } else {
+ this.blend(column, row, color, alpha);
+ }
+ }
+ }
+ }
+
+ /** Draw text whose glyph origin is the top-left corner; returns the advance in pixels. */
+ text(value: string, x: number, y: number, scale: number, color: Rgb, alpha = 1): number {
+ let cursor = x;
+ for (const character of value) {
+ const rows = glyph(character);
+ for (let row = 0; row < GLYPH_HEIGHT; row += 1) {
+ for (let column = 0; column < GLYPH_WIDTH; column += 1) {
+ if (rows[row]![column] === "#") this.fill(cursor + column * scale, y + row * scale, scale, scale, color, alpha);
+ }
+ }
+ cursor += ADVANCE * scale;
+ }
+ return cursor - x;
+ }
+
+ /** The same text stepped along a 45° line, one glyph per step; the watermark reads diagonally. */
+ diagonalText(value: string, x: number, y: number, scale: number, color: Rgb, alpha: number): void {
+ const step = GLYPH_WIDTH * scale;
+ let index = 0;
+ for (const character of value) {
+ const rows = glyph(character);
+ const originX = x + index * step;
+ const originY = y + index * step;
+ for (let row = 0; row < GLYPH_HEIGHT; row += 1) {
+ for (let column = 0; column < GLYPH_WIDTH; column += 1) {
+ if (rows[row]![column] === "#") this.fill(originX + column * scale, originY + row * scale, scale, scale, color, alpha);
+ }
+ }
+ index += 1;
+ }
+ }
+
+ png(): Uint8Array {
+ const stride = this.width * 3;
+ const scanlines = Buffer.alloc(this.height * (1 + stride));
+ for (let row = 0; row < this.height; row += 1) {
+ scanlines[row * (1 + stride)] = 0;
+ this.pixels.copy(scanlines, row * (1 + stride) + 1, row * stride, (row + 1) * stride);
+ }
+ const header = Buffer.alloc(13);
+ header.writeUInt32BE(this.width, 0);
+ header.writeUInt32BE(this.height, 4);
+ header[8] = 8; header[9] = 2; header[10] = 0; header[11] = 0; header[12] = 0;
+ return Uint8Array.from(Buffer.concat([
+ Buffer.from("\x89PNG\r\n\x1a\n", "binary"),
+ Buffer.from(pngChunk("IHDR", header)),
+ Buffer.from(pngChunk("IDAT", deflateSync(scanlines))),
+ Buffer.from(pngChunk("IEND", new Uint8Array())),
+ ]));
+ }
+}
+
+function crc32(bytes: Uint8Array): number {
+ let crc = 0xFFFFFFFF;
+ for (const byte of bytes) {
+ crc ^= byte;
+ for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0);
+ }
+ return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+function pngChunk(type: string, data: Uint8Array): Uint8Array {
+ const body = new Uint8Array(type.length + data.length);
+ for (let index = 0; index < type.length; index += 1) body[index] = type.charCodeAt(index);
+ body.set(data, type.length);
+ const result = new Uint8Array(12 + data.length);
+ const view = new DataView(result.buffer);
+ view.setUint32(0, data.length);
+ result.set(body, 4);
+ view.setUint32(8 + data.length, crc32(body));
+ return result;
+}
+
+function textWidth(value: string, scale: number): number {
+ return Math.max(0, [...value].length * ADVANCE * scale - scale);
+}
+
+/** Uppercase, one line per row, at most `maxLines` rows of `maxChars`; the last row ends in an ellipsis. */
+function wrap(value: string, maxChars: number, maxLines: number): readonly string[] {
+ const words = value.replace(/\s+/gu, " ").trim().split(" ").filter((word) => word.length > 0);
+ const lines: string[] = [];
+ let current = "";
+ for (const word of words) {
+ const candidate = current.length === 0 ? word : `${current} ${word}`;
+ if (candidate.length <= maxChars) { current = candidate; continue; }
+ if (current.length > 0) lines.push(current);
+ current = word.length > maxChars ? word.slice(0, maxChars) : word;
+ if (lines.length === maxLines) break;
+ }
+ if (current.length > 0 && lines.length < maxLines) lines.push(current);
+ const truncated = lines.join(" ").length < words.join(" ").length;
+ if (truncated && lines.length > 0) {
+ const last = lines[lines.length - 1]!;
+ lines[lines.length - 1] = `${last.slice(0, Math.max(0, maxChars - 3))}...`;
+ }
+ return lines;
+}
+
+function seconds(video: NonNullable<StandInCardRequest["video"]>): number {
+ return video.frameCount * video.frameRate.denominator / video.frameRate.numerator;
+}
+
+function timecode(frame: number, video: NonNullable<StandInCardRequest["video"]>): string {
+ const fps = video.frameRate.numerator / video.frameRate.denominator;
+ const total = frame / fps;
+ const minutes = Math.floor(total / 60);
+ const wholeSeconds = Math.floor(total - minutes * 60);
+ const frames = frame - Math.round((minutes * 60 + wholeSeconds) * fps);
+ const two = (value: number): string => String(value).padStart(2, "0");
+ return `${two(minutes)}:${two(wholeSeconds)}.${two(Math.max(0, frames))}`;
+}
+
+type Layout = {
+ readonly border: number;
+ readonly band: number;
+ readonly margin: number;
+};
+
+function layout(request: StandInCardRequest): Layout {
+ const m = Math.min(request.width, request.height);
+ const border = Math.max(6, Math.round(m * 0.03));
+ const band = request.kind === "video" ? Math.max(36, Math.round(request.height * 0.06)) : 0;
+ return { border, band, margin: border * 2 };
+}
+
+/** The still part of the card: everything but the timecode band. */
+export function drawStandInCard(request: StandInCardRequest): Uint8Array {
+ verifyStandInCardRequest(request);
+ const { width, height } = request;
+ const m = Math.min(width, height);
+ const { border, band, margin } = layout(request);
+ const raster = new Raster(width, height, GROUND[request.kind]);
+
+ // Diagonal watermark: stepped 45° lines of STAND-IN across the whole ground, faint.
+ const markScale = Math.max(2, Math.floor(m / 220));
+ const word = "STAND-IN ";
+ const wordSpan = word.length * GLYPH_WIDTH * markScale;
+ const lineGap = Math.max(GLYPH_HEIGHT * markScale * 3, Math.round(m * 0.22));
+ for (let start = -height; start < width; start += lineGap) {
+ for (let along = 0; start + along < width + wordSpan && along < width + height; along += wordSpan) {
+ raster.diagonalText(word, start + along, along, markScale, TEXT, 0.09);
+ }
+ }
+
+ // Border.
+ raster.fill(0, 0, width, border, ACCENT);
+ raster.fill(0, height - border, width, border, ACCENT);
+ raster.fill(0, 0, border, height, ACCENT);
+ raster.fill(width - border, 0, border, height, ACCENT);
+
+ // Corner tag.
+ const tagScale = Math.max(2, Math.floor(m / 260));
+ raster.text("STAND-IN", margin, margin, tagScale, ACCENT);
+ const kindTag = request.kind === "video" ? "NOT YET GENERATED" : "NOT YET GENERATED";
+ raster.text(kindTag, width - margin - textWidth(kindTag, tagScale), margin, tagScale, MUTED);
+
+ // Centre block: kind, model, frame and duration, prompt excerpt.
+ const titleScale = Math.max(4, Math.floor(m / 70));
+ const lineScale = Math.max(2, Math.floor(m / 200));
+ const promptScale = Math.max(2, Math.floor(m / 240));
+ const title = request.kind.toUpperCase();
+ const frameLine = request.video === undefined
+ ? `${width} X ${height}`
+ : `${width} X ${height} ${seconds(request.video).toFixed(2)} S ${(request.video.frameRate.numerator / request.video.frameRate.denominator).toFixed(2)} FPS`;
+ const promptLines = wrap(request.prompt, Math.max(8, Math.floor((width - margin * 2) / (ADVANCE * promptScale))), 4);
+ const blockHeight = GLYPH_HEIGHT * titleScale + lineScale * 6
+ + GLYPH_HEIGHT * lineScale * 2 + lineScale * 4
+ + (promptLines.length === 0 ? 0 : lineScale * 8 + promptLines.length * (GLYPH_HEIGHT * promptScale + promptScale * 3));
+ let y = Math.round((height - band - blockHeight) / 2);
+ const centred = (value: string, scale: number, color: Rgb): void => {
+ raster.text(value, Math.round((width - textWidth(value, scale)) / 2), y, scale, color);
+ y += GLYPH_HEIGHT * scale;
+ };
+ centred(title, titleScale, TEXT);
+ y += lineScale * 6;
+ centred(request.model, lineScale, ACCENT);
+ y += lineScale * 4;
+ centred(frameLine, lineScale, MUTED);
+ if (promptLines.length > 0) {
+ y += lineScale * 8;
+ for (const line of promptLines) {
+ centred(line, promptScale, TEXT);
+ y += promptScale * 3;
+ }
+ }
+ return raster.png();
+}
+
+/** The band a video card carries under its picture: timecode, frame counter and progress bar. */
+export function drawStandInBand(request: StandInCardRequest, frame: number): Uint8Array {
+ verifyStandInCardRequest(request);
+ const video = request.video;
+ if (video === undefined) throw new Error("only a video stand-in has a timecode band");
+ const { width } = request;
+ const { border, band, margin } = layout(request);
+ const raster = new Raster(width, band, INK);
+ raster.fill(0, 0, border, band, ACCENT);
+ raster.fill(width - border, 0, border, band, ACCENT);
+ const scale = Math.max(2, Math.floor((band - border) / (GLYPH_HEIGHT + 4)));
+ const textY = Math.round((band - border - GLYPH_HEIGHT * scale) / 2);
+ raster.text(timecode(frame, video), margin, textY, scale, TEXT);
+ const counter = `F ${frame + 1}/${video.frameCount}`;
+ raster.text(counter, width - margin - textWidth(counter, scale), textY, scale, MUTED);
+ raster.fill(0, band - border, width, border, INK);
+ raster.fill(0, band - border, Math.round(width * (frame + 1) / video.frameCount), border, ACCENT);
+ return raster.png();
+}
+
+export type StandInDrawingEnvironment = {
+ readonly ffmpegPath: string;
+ readonly processTimeoutMs: number;
+ readonly runFfmpeg: (argv: readonly string[]) => Promise<void>;
+ readonly inspectStreams: (path: string) => Promise<readonly MediaStream[]>;
+ readonly putBytes: (bytes: Uint8Array, mediaType: string) => Promise<CanonicalValue>;
+ readonly putFile: (path: string, mediaType: string) => Promise<CanonicalValue>;
+};
+
+export function standInCardNeed(value: CanonicalValue): StandInCardRequest {
+ verifyStandInCardRequest(value);
+ return value;
+}
+
+/** Draw the card; a video card is the still card with its band composited frame by frame by ffmpeg. */
+export async function renderStandInCard(
+ env: StandInDrawingEnvironment,
+ request: StandInCardRequest,
+): Promise<CanonicalValue> {
+ const still = drawStandInCard(request);
+ if (request.video === undefined) return await env.putBytes(still, "image/png");
+ const video = request.video;
+ const work = await mkdtemp(join(tmpdir(), "hypit-stand-in-"));
+ try {
+ const base = join(work, "card.png");
+ await writeFile(base, still);
+ for (let frame = 0; frame < video.frameCount; frame += 1) {
+ await writeFile(join(work, `band-${String(frame).padStart(6, "0")}.png`), drawStandInBand(request, frame));
+ }
+ const fps = `${video.frameRate.numerator}/${video.frameRate.denominator}`;
+ const output = join(work, "card.mp4");
+ const silence = video.audio === "silence";
+ await env.runFfmpeg([
+ "-y",
+ "-loop", "1", "-framerate", fps, "-i", base,
+ "-framerate", fps, "-i", join(work, "band-%06d.png"),
+ ...(silence ? ["-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo"] : []),
+ "-filter_complex", "[0:v][1:v]overlay=0:main_h-overlay_h:shortest=1,format=yuv420p[v]",
+ "-map", "[v]", ...(silence ? ["-map", "2:a:0", "-c:a", "aac", "-b:a", "96k", "-shortest"] : []),
+ "-frames:v", String(video.frameCount), "-r", fps, "-fps_mode", "cfr",
+ "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
+ "-movflags", "+faststart", output,
+ ]);
+ const streams = await env.inspectStreams(output);
+ const visual = streams.find((item) => item.kind === "video");
+ const audio = streams.filter((item) => item.kind === "audio");
+ if (visual === undefined || visual.kind !== "video" || visual.decodedUnitCount !== video.frameCount
+ || visual.width !== request.width || visual.height !== request.height
+ || streams.length !== 1 + audio.length || audio.length !== (silence ? 1 : 0)) {
+ throw new Error("Stand-in video differs from its requested frame domain");
+ }
+ return await env.putFile(output, "video/mp4");
+ } finally {
+ await rm(work, { recursive: true, force: true }).catch(() => {});
+ }
+}
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index 267e7204..47d2c97f 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -32,6 +32,7 @@ import {
} from "@hypit/protocol";
import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
+import { renderStandInCard, standInCardNeed } from "./card.js";
import { parseMediaInspection } from "./probe.js";
import {
compositeAnimatedWebpFrame,
@@ -786,6 +787,41 @@ export async function executeRenderStillVideo(
}
}
+/** Draw a stand-in card for a generated output that has not been generated; a Provider draws, it never generates. */
+export async function executeDrawStandInCard(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ const request = standInCardNeed(constraints);
+ const value = await renderStandInCard({
+ ffmpegPath: env.ffmpegPath,
+ processTimeoutMs: env.processTimeoutMs,
+ runFfmpeg: async (argv) => {
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv,
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ },
+ inspectStreams: async (path) => {
+ const inspected = await outputInspection({
+ path,
+ mediaType: "video/mp4",
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ return inspected.streams;
+ },
+ putBytes: async (bytes, mediaType) => await env.artifacts.put(bytes, mediaType) as unknown as CanonicalValue,
+ putFile: async (path, mediaType) => await env.artifacts.putFile(path, mediaType) as unknown as CanonicalValue,
+ }, request);
+ return artifactResult(value as unknown as BlobRef);
+}
+
type TransformPlan = {
readonly videoFilters: readonly string[];
readonly audioFilters: readonly string[];
diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.ts
index fcc10457..6dbe5f05 100644
--- a/packages/media-execution/src/index.ts
+++ b/packages/media-execution/src/index.ts
@@ -7,8 +7,10 @@ export {
executeProjectSpeechEvidenceAudio,
executeRenderTimelineAudio,
executeRenderStillVideo,
+ executeDrawStandInCard,
executeTransformMedia,
} from "./execute.js";
+export { drawStandInBand, drawStandInCard } from "./card.js";
export { parseMediaInspection } from "./probe.js";
export { verifyCompositableSurfaceBytes } from "./surface.js";
export { probeMediaToolchain } from "./toolchain.js";
diff --git a/packages/media-track/preview/preview.svml b/packages/media-track/preview/preview.svml
index 677cf647..3fd79c7e 100644
--- a/packages/media-track/preview/preview.svml
+++ b/packages/media-track/preview/preview.svml
@@ -9,14 +9,18 @@
The checked-in Track.png is the package catalogue asset. This Source and its ordinary Run
preserve how it was authored; regenerating the encoded frame is a normal Build.
- A placeholder shows the Frame treatment and nothing inside it. To show the component holding a
- picture, commit a sample beside this file and satisfy the generated output in `build.svrun`;
- the Profile's local media Endpoint still inspects and normalizes it, in Studio as in a Build:
+ A placeholder shows the Frame treatment and nothing inside it. To look at the component before
+ buying its pictures, stand in for each generation in the Run; the card is derived from the
+ model's own draft, and the Profile's local media Endpoint normalizes it, in Studio as in a Build:
- <file id="sample-a" type="@hypit/artifact@1#BlobArtifact" from="./sample-a.mp4" media-type="video/mp4"/>
- <satisfy output="motion-a.video" candidate="sample-a"/>
+ <import from="@hypit/seedance@1" as="seedance"/>
+ <fragment id="a-card" using="seedance:mini-stand-in">
+ <input name="draft" from="motion-a.draft"/><input name="prompt" from="direction"/><input name="clock" from="clock"/>
+ </fragment>
+ <satisfy output="motion-a.video" candidate="a-card.video"/>
- Candidate selection is explicit in the Run; Studio never replaces missing media itself. -->
+ A committed sample works the same way as a `file` Candidate on `motion-a.video`. Candidate
+ selection is explicit in the Run; Studio never replaces missing media itself. -->
<svml>
<import from="@hypit/script@1"/>
<import as="wording" from="@hypit/text@1"/>
diff --git a/packages/minimax-h3/src/activation.ts b/packages/minimax-h3/src/activation.ts
index 7229333a..e253c792 100644
--- a/packages/minimax-h3/src/activation.ts
+++ b/packages/minimax-h3/src/activation.ts
@@ -4,6 +4,7 @@ import { minimaxH3Component, minimaxH3Definition, minimaxH3Manifest, minimaxH3Mo
import { decodeMinimaxFrameVideoSurface, decodeMinimaxReferenceVideoSurface, decodeMinimaxTextVideoSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: minimaxH3Manifest }], components: [minimaxH3Component], hostFacets: [
minimaxH3Definition.hostFacet,
+ ...(minimaxH3Definition.runFragmentFacet === undefined ? [] : [minimaxH3Definition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,
declaration: minimaxH3MarkupSurfaces.find((item) => item.name === "text-video")!, handler: decodeMinimaxTextVideoSurface }),
createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,
diff --git a/packages/model-kit/package.json b/packages/model-kit/package.json
index f1bae183..3132e021 100644
--- a/packages/model-kit/package.json
+++ b/packages/model-kit/package.json
@@ -13,7 +13,10 @@
"@hypit/elaborator": "workspace:*",
"@hypit/generation": "workspace:*",
"@hypit/host": "workspace:*",
+ "@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
+ "@hypit/run": "workspace:*",
+ "@hypit/stand-in": "workspace:*",
"@hypit/text": "workspace:*"
}
}
diff --git a/packages/model-kit/src/index.ts b/packages/model-kit/src/index.ts
index 64018f9d..395adff5 100644
--- a/packages/model-kit/src/index.ts
+++ b/packages/model-kit/src/index.ts
@@ -27,7 +27,15 @@ import type {
GenerationPortTable,
GenerationRequestDraft,
} from "@hypit/generation";
+import { programSpaceDependency } from "@hypit/program-space";
+import type { ProgramClock } from "@hypit/program-space";
+import { createRunFragmentHostFacet } from "@hypit/run";
+import type { RunFragmentHostFacet } from "@hypit/run";
+import { standInDependency } from "@hypit/stand-in";
import { textDependency, textTypes } from "@hypit/text";
+
+import { exactModelStandIn, standInCardRequestFromDraft, standInNeed } from "./stand-in.js";
+import type { ExactModelStandIn } from "./stand-in.js";
import type { Text } from "@hypit/text";
import {
canonicalize,
@@ -87,6 +95,8 @@ export type ExactModelEndpoint = {
readonly textBindings: Readonly<Record<string, ExactModelTextBindingEndpoint>>;
readonly ports: GenerationPortTable;
readonly fragment: ReturnType<typeof sealGraphFragment>;
+ /** The drawn stand-in for this model's output; absent for a model whose output is not a picture. */
+ readonly standIn?: ExactModelStandIn;
};
export type ExactModelMediaBindingEndpoint = {
@@ -125,6 +135,8 @@ export type ExactModelModule<Key extends string = string> = {
};
/** Inert declaration of these exact models for Hosts that address one directly. */
readonly hostFacet: ExactModelHostFacet;
+ /** Run Fragments that stand in for this module's picture outputs, one per model, named `<key>-stand-in`. */
+ readonly runFragmentFacet?: RunFragmentHostFacet;
};
export const exactModelHostAbi = "hypit.exact-model-host@1";
@@ -372,7 +384,11 @@ export function defineExactModelModule<const Key extends string>(
assert(new Set(keys).size === keys.length, `${options.module.name} repeats an endpoint key`);
const models = options.endpoints.map((item) => item.ports.model);
assert(new Set(models).size === models.length, `${options.module.name} repeats an exact model`);
- const endpointData = options.endpoints.map((spec) => ({ spec, ...endpointRef(options.module, spec) }));
+ const endpointData = options.endpoints.map((spec) => {
+ const refs = endpointRef(options.module, spec);
+ const standIn = exactModelStandIn({ module: options.module, producerName: spec.producerName, draftType: refs.draftType, ports: spec.ports });
+ return { spec, ...refs, ...(standIn === undefined ? {} : { standIn }) };
+ });
const manifest: ModuleManifest = {
format: "hypit.module@1",
@@ -382,6 +398,8 @@ export function defineExactModelModule<const Key extends string>(
{ module: generationModuleRef },
artifactDependency,
textDependency,
+ standInDependency,
+ programSpaceDependency,
],
types: endpointData.flatMap((item) => [
{
@@ -437,6 +455,12 @@ export function defineExactModelModule<const Key extends string>(
outputs: [{ name: "request", type: item.requestType }],
needs: [],
},
+ ...(item.standIn === undefined ? [] : [{
+ name: item.standIn.producer.name,
+ inputs: item.standIn.inputs,
+ outputs: [],
+ needs: [standInNeed],
+ }]),
]),
};
@@ -467,9 +491,13 @@ export function defineExactModelModule<const Key extends string>(
textBindings: item.textBindings,
ports: item.spec.ports,
fragment,
+ ...(item.standIn === undefined ? {} : { standIn: item.standIn }),
} satisfies ExactModelEndpoint];
}));
+ const standIns = endpointData.flatMap((item) =>
+ item.standIn === undefined ? [] : [[`${item.spec.key}-stand-in`, item.standIn.fragment] as const]);
+
return {
module: { ...options.module },
manifest,
@@ -551,8 +579,24 @@ export function defineExactModelModule<const Key extends string>(
needs: {},
}),
},
+ ...(item.standIn === undefined ? [] : [{
+ producer: item.standIn.producer,
+ handler: ({ inputs }: ProducerHandlerContext) => {
+ const draft = inlineValue<GenerationRequestDraft>(inputs.draft!.value, `${item.spec.key} draft`);
+ const prompt = inlineValue<Text>(inputs.prompt!.value, `${item.spec.key} stand-in prompt`);
+ const clock = inputs.clock === undefined ? undefined : inlineValue<ProgramClock>(inputs.clock.value, `${item.spec.key} stand-in clock`);
+ const request = standInCardRequestFromDraft({ ports: item.spec.ports, draft, prompt, ...(clock === undefined ? {} : { clock }) });
+ return { outputs: {}, needs: { card: canonicalize(request as unknown as CanonicalValue) } };
+ },
+ }]),
]),
},
+ ...(standIns.length === 0 ? {} : {
+ runFragmentFacet: createRunFragmentHostFacet({
+ name: `${options.module.name}@${options.module.version}`,
+ fragments: Object.fromEntries(standIns),
+ }),
+ }),
};
}
@@ -664,3 +708,5 @@ export function createExactModelPrimaryGenerationFragment(
}],
});
}
+export { standInCardRequestFromDraft, standInFrameFromDraft } from "./stand-in.js";
+export type { ExactModelStandIn } from "./stand-in.js";
diff --git a/packages/model-kit/src/stand-in.ts b/packages/model-kit/src/stand-in.ts
new file mode 100644
index 00000000..c7f60585
--- /dev/null
+++ b/packages/model-kit/src/stand-in.ts
@@ -0,0 +1,139 @@
+import { artifactTypes } from "@hypit/artifact";
+import { sealGraphFragment } from "@hypit/elaborator";
+import type { GenerationPortTable, GenerationRequestDraft } from "@hypit/generation";
+import { assertProgramClockIdentity, programSpaceTypes } from "@hypit/program-space";
+import type { ProgramClock } from "@hypit/program-space";
+import type { ModuleRef, ProducerRef, TypeRef } from "@hypit/protocol";
+import { sealStandInCardRequest, standInCapabilities } from "@hypit/stand-in";
+import type { StandInCardRequest } from "@hypit/stand-in";
+import { textTypes } from "@hypit/text";
+import type { Text } from "@hypit/text";
+
+/**
+ * A stand-in for an exact model's output is derived from that model's own request draft: the
+ * frame from its resolution and aspect ports, the duration from its duration port, the model from
+ * its port table, the prompt from the same Text edge the model would read. Nothing is restated by
+ * hand, so the card follows the Source. What cannot be derived is refused with the reason.
+ */
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+/** How many pixels a resolution token names, and on which side. */
+const RESOLUTIONS: Readonly<Record<string, { readonly pixels: number; readonly side: "shorter" | "longer" }>> = {
+ "480p": { pixels: 480, side: "shorter" },
+ "720p": { pixels: 720, side: "shorter" },
+ "1080p": { pixels: 1080, side: "shorter" },
+ "4k": { pixels: 2160, side: "shorter" },
+ "1K": { pixels: 1024, side: "longer" },
+ "2K": { pixels: 2048, side: "longer" },
+ "4K": { pixels: 4096, side: "longer" },
+};
+
+function single(draft: GenerationRequestDraft, port: string): string | number | boolean | undefined {
+ const values = draft.ports[port];
+ if (values === undefined || values.length === 0) return undefined;
+ const value = values[0];
+ return typeof value === "object" ? undefined : value;
+}
+
+function even(value: number): number {
+ const rounded = Math.round(value);
+ return rounded % 2 === 0 ? rounded : rounded + 1;
+}
+
+export function standInFrameFromDraft(ports: GenerationPortTable, draft: GenerationRequestDraft): { readonly width: number; readonly height: number } {
+ const size = single(draft, "size");
+ if (typeof size === "string") {
+ const match = /^(\d+)\s*[xX×]\s*(\d+)$/u.exec(size.trim());
+ assert(match !== null, `${ports.model} size port "${size}" is not WIDTHxHEIGHT; no stand-in frame can be derived`);
+ return { width: even(Number(match[1])), height: even(Number(match[2])) };
+ }
+ const resolution = single(draft, "resolution");
+ assert(typeof resolution === "string" && RESOLUTIONS[resolution] !== undefined,
+ `${ports.model} names no resolution a stand-in understands (${String(resolution)}); the card cannot know its frame`);
+ const aspect = single(draft, "aspectRatio");
+ let ratio: { readonly w: number; readonly h: number };
+ if (typeof aspect === "string") {
+ const match = /^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/u.exec(aspect.trim());
+ assert(match !== null, `${ports.model} aspectRatio "${aspect}" is not W:H; no stand-in frame can be derived`);
+ ratio = { w: Number(match[1]), h: Number(match[2]) };
+ } else {
+ assert(ports.result === "image", `${ports.model} declares no aspect ratio; a video stand-in cannot guess its frame`);
+ ratio = { w: 1, h: 1 };
+ }
+ const { pixels, side } = RESOLUTIONS[resolution]!;
+ const portrait = ratio.h > ratio.w;
+ const fixShorter = side === "shorter";
+ // The named pixel count lands on the shorter side (480p, 720p…) or the longer side (1K, 2K…).
+ const widthIsFixed = fixShorter ? portrait : !portrait;
+ return widthIsFixed
+ ? { width: even(pixels), height: even(pixels * ratio.h / ratio.w) }
+ : { width: even(pixels * ratio.w / ratio.h), height: even(pixels) };
+}
+
+export function standInCardRequestFromDraft(args: {
+ readonly ports: GenerationPortTable;
+ readonly draft: GenerationRequestDraft;
+ readonly prompt: Text;
+ readonly clock?: ProgramClock;
+}): StandInCardRequest {
+ const { ports, draft } = args;
+ assert(ports.result !== "audio", `${ports.model} produces audio; a stand-in card is a picture`);
+ const frame = standInFrameFromDraft(ports, draft);
+ const base = { ...frame, model: ports.model, prompt: args.prompt.value };
+ if (ports.result === "image") return sealStandInCardRequest({ kind: "image", ...base });
+ const clock = args.clock;
+ assert(clock !== undefined, `a video stand-in for ${ports.model} needs the piece's clock`);
+ assertProgramClockIdentity(clock);
+ const duration = single(draft, "duration");
+ assert(typeof duration === "number" && Number.isFinite(duration) && duration > 0,
+ `${ports.model} has no fixed duration (${String(duration)}); write the seconds, an automatic duration has no stand-in`);
+ const frameCount = Math.round(duration * clock.frameRate.numerator / clock.frameRate.denominator);
+ assert(Number.isSafeInteger(frameCount) && frameCount > 0, `${ports.model} duration yields no frames on this clock`);
+ const audio = single(draft, "generateAudio") === true;
+ return sealStandInCardRequest({ kind: "video", ...base, video: { frameRate: clock.frameRate, frameCount, ...(audio ? { audio: "silence" as const } : {}) } });
+}
+
+export type ExactModelStandIn = {
+ readonly producer: ProducerRef;
+ readonly fragment: ReturnType<typeof sealGraphFragment>;
+ /** The export name a Run satisfies with: `image` or `video`, the Blob the model would have made. */
+ readonly output: "image" | "video";
+ readonly inputs: readonly { readonly name: string; readonly type: TypeRef }[];
+};
+
+/** The stand-in shape for one endpoint, or none for a model whose output is not a picture. */
+export function exactModelStandIn(args: {
+ readonly module: ModuleRef;
+ readonly producerName: string;
+ readonly draftType: TypeRef;
+ readonly ports: GenerationPortTable;
+}): ExactModelStandIn | undefined {
+ if (args.ports.result === "audio") return undefined;
+ const video = args.ports.result === "video";
+ const producer = { module: args.module, name: `stand-in-${args.producerName}` };
+ const inputs = [
+ { name: "draft", type: args.draftType },
+ { name: "prompt", type: textTypes.text },
+ ...(video ? [{ name: "clock", type: programSpaceTypes.clock }] : []),
+ ];
+ const fragment = sealGraphFragment({
+ inputs,
+ operations: [{
+ id: "card",
+ producer,
+ inputs: Object.fromEntries(inputs.map((input) => [input.name, { kind: "fragment-input" as const, name: input.name }])),
+ result: { kind: "need", name: "card" },
+ }],
+ exports: [{
+ name: video ? "video" : "image",
+ type: artifactTypes.blob,
+ root: { kind: "fragment-operation", operation: "card" },
+ }],
+ });
+ return { producer, fragment, output: video ? "video" : "image", inputs };
+}
+
+export const standInNeed = { name: "card", capability: standInCapabilities.drawCard, returns: artifactTypes.blob } as const;
diff --git a/packages/model-kit/test/stand-in.test.ts b/packages/model-kit/test/stand-in.test.ts
new file mode 100644
index 00000000..7242dcff
--- /dev/null
+++ b/packages/model-kit/test/stand-in.test.ts
@@ -0,0 +1,85 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { sealGenerationRequestDraft } from "@hypit/generation";
+import type { GenerationPortTable } from "@hypit/generation";
+import { sealText } from "@hypit/text";
+
+import { defineExactModelModule, standInCardRequestFromDraft, standInFrameFromDraft } from "@hypit/model-kit";
+
+const clip: GenerationPortTable = {
+ model: "clip-model",
+ result: "video",
+ ports: [
+ { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },
+ { name: "resolution", value: { kind: "enum", values: ["480p", "720p", "1080p"] }, minItems: 1, maxItems: 1 },
+ { name: "aspectRatio", value: { kind: "enum", values: ["9:16", "16:9", "1:1"] }, minItems: 1, maxItems: 1 },
+ { name: "duration", value: { kind: "enum", values: [-1, 4, 5, 8] }, minItems: 1, maxItems: 1 },
+ { name: "generateAudio", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },
+ ],
+ requires: [],
+};
+const picture: GenerationPortTable = {
+ model: "picture-model",
+ result: "image",
+ ports: [
+ { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },
+ { name: "resolution", value: { kind: "enum", values: ["1K", "2K"] }, minItems: 1, maxItems: 1 },
+ { name: "aspectRatio", value: { kind: "enum", values: ["3:2", "1:1"] }, minItems: 0, maxItems: 1 },
+ ],
+ requires: [],
+};
+const voice: GenerationPortTable = {
+ model: "voice-model",
+ result: "audio",
+ ports: [{ name: "text", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 }],
+ requires: [],
+};
+const clock = { frameRate: { numerator: 30, denominator: 1 } };
+
+test("a stand-in card is derived from the model's draft: frame, duration, model and prompt", () => {
+ const draft = sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] });
+ const request = standInCardRequestFromDraft({ ports: clip, draft, prompt: sealText("A quiet kitchen"), clock });
+ assert.deepEqual(request, {
+ kind: "video", width: 720, height: 1280, model: "clip-model", prompt: "A quiet kitchen",
+ video: { frameRate: { numerator: 30, denominator: 1 }, frameCount: 150 },
+ });
+ const spoken = standInCardRequestFromDraft({ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5], generateAudio: [true] }), prompt: sealText("x"), clock });
+ assert.equal(spoken.video?.audio, "silence", "a model that would have spoken leaves a silent track to cut against");
+ assert.deepEqual(standInFrameFromDraft(clip, sealGenerationRequestDraft(clip, { resolution: ["1080p"], aspectRatio: ["16:9"], duration: [4] })),
+ { width: 1920, height: 1080 });
+ assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["1K"] })), { width: 1024, height: 1024 });
+ assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["2K"], aspectRatio: ["3:2"] })), { width: 2048, height: 1366 });
+});
+
+test("what a stand-in cannot derive it refuses with the reason", () => {
+ assert.throws(() => standInCardRequestFromDraft({
+ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [-1] }),
+ prompt: sealText("x"), clock,
+ }), /automatic duration has no stand-in/u);
+ assert.throws(() => standInCardRequestFromDraft({
+ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] }), prompt: sealText("x"),
+ }), /needs the piece's clock/u);
+ assert.throws(() => standInCardRequestFromDraft({
+ ports: voice, draft: sealGenerationRequestDraft(voice, {}), prompt: sealText("x"),
+ }), /produces audio/u);
+});
+
+test("an exact model module exports one stand-in Run Fragment per picture model and none for audio", () => {
+ const module = defineExactModelModule({
+ module: { name: "@example/models", version: "1" },
+ endpoints: [
+ { key: "clip", requestTypeName: "ClipRequest", producerName: "request-clip", ports: clip },
+ { key: "picture", requestTypeName: "PictureRequest", producerName: "request-picture", ports: picture },
+ { key: "voice", requestTypeName: "VoiceRequest", producerName: "request-voice", ports: voice },
+ ],
+ });
+ assert.deepEqual(Object.keys(module.runFragmentFacet!.implementation.fragments).sort(), ["clip-stand-in", "picture-stand-in"]);
+ assert.equal(module.runFragmentFacet!.offers[0], "@example/models@1");
+ assert.deepEqual(module.endpoints.clip.standIn!.inputs.map((input) => input.name), ["draft", "prompt", "clock"]);
+ assert.deepEqual(module.endpoints.picture.standIn!.inputs.map((input) => input.name), ["draft", "prompt"]);
+ assert.equal(module.endpoints.voice.standIn, undefined);
+ assert.ok(module.manifest.producers.some((item) => item.name === "stand-in-request-clip"
+ && item.needs[0]?.capability.name === "draw-card"));
+ assert.ok(module.manifest.dependencies.some((item) => item.module.name === "@hypit/stand-in"));
+});
diff --git a/packages/nano-banana/src/activation.ts b/packages/nano-banana/src/activation.ts
index ff2c6ea4..f8bd921c 100644
--- a/packages/nano-banana/src/activation.ts
+++ b/packages/nano-banana/src/activation.ts
@@ -6,6 +6,7 @@ import {
import { decodeNanoBananaImageSurface, decodeNanoBananaProImageSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: nanoBananaManifest }], components: [nanoBananaComponent], hostFacets: [
nanoBananaDefinition.hostFacet,
+ ...(nanoBananaDefinition.runFragmentFacet === undefined ? [] : [nanoBananaDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,
declaration: nanoBananaMarkupSurfaces.find((item) => item.name === "image")!, handler: decodeNanoBananaImageSurface }),
createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,
diff --git a/packages/provider-media-local/package.json b/packages/provider-media-local/package.json
index a7d6fe7f..755c9d2c 100644
--- a/packages/provider-media-local/package.json
+++ b/packages/provider-media-local/package.json
@@ -20,7 +20,8 @@
"@hypit/runtime": "workspace:*",
"@hypit/runtime-kit": "workspace:*",
"@hypit/runtime-host-node": "workspace:*",
- "@hypit/speech": "workspace:*"
+ "@hypit/speech": "workspace:*",
+ "@hypit/stand-in": "workspace:*"
},
"devDependencies": {
"@hypit/driver-node": "workspace:*"
diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.ts
index f85e8b57..b231a521 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/provider-media-local/src/provider.ts
@@ -9,10 +9,12 @@ import {
executeProjectSpeechEvidenceAudio,
executeRenderTimelineAudio,
executeRenderStillVideo,
+ executeDrawStandInCard,
executeTransformMedia,
} from "@hypit/media-execution";
import type { MediaExecutionEnvironment, MediaOperationResult } from "@hypit/media-execution";
import { mediaPipelineCapabilities } from "@hypit/media-pipeline";
+import { standInCapabilities } from "@hypit/stand-in";
import { isStreamingResourceStore } from "@hypit/runtime";
import { speechTypes } from "@hypit/speech";
import { defineEndpointPackage } from "@hypit/endpoint-kit";
@@ -119,6 +121,12 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
returns: artifactTypes.blob,
handler: operation(executeRenderStillVideo),
},
+ {
+ lifecycle: "immediate" as const,
+ capability: standInCapabilities.drawCard,
+ returns: artifactTypes.blob,
+ handler: operation(executeDrawStandInCard),
+ },
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,
diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.ts
index 0f1bf96b..bbf87a17 100644
--- a/packages/provider-media-local/test/provider.test.ts
+++ b/packages/provider-media-local/test/provider.test.ts
@@ -22,6 +22,7 @@ import {
selectMediaStreams,
} from "@hypit/media-pipeline";
import { canonicalize } from "@hypit/protocol";
+import { standInCapabilities } from "@hypit/stand-in";
import type { CapabilityRef, CanonicalValue, Need, TypeRef } from "@hypit/protocol";
import { createLocalMediaProvider } from "../src/index.js";
@@ -827,3 +828,57 @@ test("local media Provider executes an end-aligned loop from the exact authored
await rm(root, { recursive: true, force: true });
}
});
+
+function pngSize(bytes: Uint8Array): { readonly width: number; readonly height: number } {
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ assert.equal(Buffer.from(bytes.subarray(1, 4)).toString("ascii"), "PNG");
+ return { width: view.getUint32(16), height: view.getUint32(20) };
+}
+
+test("local media Provider draws a stand-in card as a picture and as an exact silent clip", {
+ skip: !hasMediaBinaries && "ffmpeg is not installed",
+}, async () => {
+ const resources = new MemoryResourceStore();
+ const fulfillBlob = async (request: Need): Promise<Awaited<ReturnType<MemoryResourceStore["put"]>>> => {
+ const provider = await handlerFor(request);
+ const result = await provider.handler({
+ command: { kind: "fulfill-need", id: `command:${request.id}`, need: request },
+ need: request,
+ resources,
+ credentials: {},
+ });
+ assert.equal(result.value.kind, "blob");
+ return result.value as Awaited<ReturnType<MemoryResourceStore["put"]>>;
+ };
+ const picture = await fulfillBlob(need("need:card-image", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
+ kind: "image", width: 640, height: 400, model: "picture-model", prompt: "A quiet kitchen counter with fresh basil",
+ })));
+ assert.equal(picture.kind, "blob");
+ assert.equal(picture.mediaType, "image/png");
+ const bytes = await resources.get(picture.resource);
+ assert.ok(bytes !== undefined);
+ assert.deepEqual(pngSize(bytes), { width: 640, height: 400 });
+
+ const clip = await fulfillBlob(need("need:card-video", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
+ kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Locked medium close-up",
+ video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12 },
+ })));
+ assert.equal(clip.mediaType, "video/mp4");
+ const inspection = await inspectArtifact(resources, clip);
+ const visual = inspection.streams.find((item) => item.kind === "video");
+ assert.ok(visual !== undefined && visual.kind === "video");
+ assert.equal(inspection.streams.length, 1, "a stand-in clip carries no sound unless the model would have");
+ assert.equal(visual.width, 320);
+ assert.equal(visual.height, 568);
+ assert.equal(visual.decodedUnitCount, 12);
+
+ const spoken = await fulfillBlob(need("need:card-spoken", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
+ kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Spoken dialogue",
+ video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12, audio: "silence" },
+ })));
+ const spokenStreams = (await inspectArtifact(resources, spoken)).streams;
+ const sound = spokenStreams.find((item) => item.kind === "audio");
+ assert.ok(sound !== undefined && sound.kind === "audio", "a model that would have spoken leaves a silent track");
+ assert.equal(sound.sampleRate, 48_000);
+ assert.equal(spokenStreams.length, 2);
+});
diff --git a/packages/seedance/src/activation.ts b/packages/seedance/src/activation.ts
index c4bda476..912cb520 100644
--- a/packages/seedance/src/activation.ts
+++ b/packages/seedance/src/activation.ts
@@ -17,6 +17,7 @@ export const hypitPackage = {
components: [seedanceComponent],
hostFacets: [
seedanceDefinition.hostFacet,
+ ...(seedanceDefinition.runFragmentFacet === undefined ? [] : [seedanceDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: seedanceModuleRef,
declaration: seedanceMarkupSurfaces.find((item) => item.name === "text-video")!,
diff --git a/packages/seedream/src/activation.ts b/packages/seedream/src/activation.ts
index 2ccbedad..1bccf402 100644
--- a/packages/seedream/src/activation.ts
+++ b/packages/seedream/src/activation.ts
@@ -4,6 +4,7 @@ import { seedreamComponent, seedreamDefinition, seedreamManifest, seedreamModule
import { decodeSeedreamReferenceImageSurface, decodeSeedreamTextImageSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: seedreamManifest }], components: [seedreamComponent], hostFacets: [
seedreamDefinition.hostFacet,
+ ...(seedreamDefinition.runFragmentFacet === undefined ? [] : [seedreamDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: seedreamModuleRef,
declaration: seedreamMarkupSurfaces.find((item) => item.name === "text-image")!, handler: decodeSeedreamTextImageSurface }),
createMarkupSurfaceHostFacet({ module: seedreamModuleRef,
diff --git a/packages/stand-in/README.md b/packages/stand-in/README.md
new file mode 100644
index 00000000..d06f5082
--- /dev/null
+++ b/packages/stand-in/README.md
@@ -0,0 +1,29 @@
+# `@hypit/stand-in`
+
+A stand-in is what a generated output looks like before it is generated: a designed card, drawn
+instead of bought, that the author selects as a Candidate for that output in an ordinary Run. The
+card is derived from the model's own request — kind, frame, model, duration, prompt — so it follows
+the Source when the Source changes, and it goes through the same inspect and normalize steps as the
+real output would.
+
+This module owns the vocabulary only: the `StandInCardRequest` type and the `draw-card` capability.
+Every exact-model package built with `@hypit/model-kit` exports one Run Fragment per model that
+turns its draft into that request; a local media Provider draws the card (`@hypit/media-execution`).
+Nothing here is a mock: the Provider that serves `draw-card` really draws a card, prices it as
+local work, and never pretends to be the model.
+
+```xml
+<import from="@hypit/seedance@1" as="seedance"/>
+<fragment id="kitchen-card" using="seedance:mini-stand-in">
+ <input name="draft" from="kitchen.draft"/>
+ <input name="prompt" from="kitchen-direction"/>
+ <input name="clock" from="clock"/>
+</fragment>
+<satisfy output="kitchen.video" candidate="kitchen-card.video"/>
+```
+
+Remove the `satisfy` line and build again to buy the real output. The card carries a diagonal
+STAND-IN watermark, its kind in large type, the model, the frame and duration, an excerpt of the
+prompt, and — for video — a running timecode and progress bar, so a cut that lands on the wrong
+frame is visible in the preview. Text on the card is drawn with a built-in bitmap font: Latin
+letters, digits and common punctuation; other characters appear as boxes.
diff --git a/packages/stand-in/package.json b/packages/stand-in/package.json
new file mode 100644
index 00000000..2966fa97
--- /dev/null
+++ b/packages/stand-in/package.json
@@ -0,0 +1,19 @@
+{
+ "name": "@hypit/stand-in",
+ "version": "0.0.0-dev",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@hypit/artifact": "workspace:*",
+ "@hypit/component-kit": "workspace:*",
+ "@hypit/media": "workspace:*",
+ "@hypit/protocol": "workspace:*"
+ }
+}
diff --git a/packages/stand-in/src/activation.ts b/packages/stand-in/src/activation.ts
new file mode 100644
index 00000000..d38455b1
--- /dev/null
+++ b/packages/stand-in/src/activation.ts
@@ -0,0 +1,10 @@
+import { standInComponent, standInManifest } from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.node-package@1" as const,
+ modules: [{ manifest: standInManifest }],
+ components: [standInComponent],
+ hostFacets: [],
+};
+
+export default hypitPackage;
diff --git a/packages/stand-in/src/index.ts b/packages/stand-in/src/index.ts
new file mode 100644
index 00000000..0be29e75
--- /dev/null
+++ b/packages/stand-in/src/index.ts
@@ -0,0 +1,87 @@
+import type { ComponentPackage } from "@hypit/component-kit";
+import type { MediaRational } from "@hypit/media";
+import { canonicalize } from "@hypit/protocol";
+import type { CanonicalValue, StoredValue } from "@hypit/protocol";
+
+import { standInTypes } from "./manifest.js";
+
+export {
+ standInCapabilities,
+ standInDependency,
+ standInManifest,
+ standInModuleRef,
+ standInTypes,
+} from "./manifest.js";
+
+/**
+ * What a stand-in card says about the output it stands in for. Every field is derived from the
+ * declaration the card replaces — the model's own request — never restated by hand, so the card
+ * follows the Source when the Source changes.
+ */
+export type StandInCardRequest = {
+ readonly kind: "image" | "video";
+ /** Even, positive pixel dimensions of the card, the frame the real output would have. */
+ readonly width: number;
+ readonly height: number;
+ /** The exact model that would have generated the output. */
+ readonly model: string;
+ /** The prompt the model would have received; the card prints an excerpt. */
+ readonly prompt: string;
+ /** A video card runs for exactly this many frames at this clock and carries a running timecode. */
+ readonly video?: {
+ readonly frameRate: MediaRational;
+ readonly frameCount: number;
+ /** The model would have generated sound, so the card carries a silent 48 kHz stereo track. */
+ readonly audio?: "silence";
+ };
+};
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function evenPositive(value: unknown, subject: string): number {
+ assert(typeof value === "number" && Number.isSafeInteger(value) && value > 0 && value % 2 === 0,
+ `${subject} must be a positive even integer`);
+ return value;
+}
+
+export function verifyStandInCardRequest(value: unknown): asserts value is StandInCardRequest {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "StandInCardRequest must be an object");
+ const item = value as Record<string, unknown>;
+ assert(item.kind === "image" || item.kind === "video", "StandInCardRequest kind must be image or video");
+ evenPositive(item.width, "StandInCardRequest width");
+ evenPositive(item.height, "StandInCardRequest height");
+ assert(typeof item.model === "string" && item.model.trim().length > 0, "StandInCardRequest model must name the model");
+ assert(typeof item.prompt === "string", "StandInCardRequest prompt must be a string");
+ if (item.kind === "video") {
+ const video = item.video as Record<string, unknown> | undefined;
+ assert(video !== undefined && typeof video === "object", "a video StandInCardRequest needs its frame domain");
+ const rate = video.frameRate as Record<string, unknown> | undefined;
+ assert(rate !== undefined && Number.isSafeInteger(rate.numerator) && (rate.numerator as number) > 0
+ && Number.isSafeInteger(rate.denominator) && (rate.denominator as number) > 0,
+ "StandInCardRequest frameRate must be a positive rational");
+ assert(Number.isSafeInteger(video.frameCount) && (video.frameCount as number) > 0,
+ "StandInCardRequest frameCount must be a positive integer");
+ assert(video.audio === undefined || video.audio === "silence", "StandInCardRequest audio can only be silence");
+ } else {
+ assert(item.video === undefined, "an image StandInCardRequest has no frame domain");
+ }
+}
+
+export function sealStandInCardRequest(value: StandInCardRequest): StandInCardRequest {
+ verifyStandInCardRequest(value);
+ return canonicalize(value as unknown as CanonicalValue) as unknown as StandInCardRequest;
+}
+
+function inline(value: StoredValue, subject: string): unknown {
+ assert(value.kind === "inline", `${subject} must be inline`);
+ return value.value;
+}
+
+export const standInComponent = {
+ validators: [{
+ type: standInTypes.cardRequest,
+ handler: ({ value }) => { verifyStandInCardRequest(inline(value, "StandInCardRequest")); },
+ }],
+} satisfies ComponentPackage;
diff --git a/packages/stand-in/src/manifest.ts b/packages/stand-in/src/manifest.ts
new file mode 100644
index 00000000..0a1f5b18
--- /dev/null
+++ b/packages/stand-in/src/manifest.ts
@@ -0,0 +1,24 @@
+import { artifactDependency, artifactTypes } from "@hypit/artifact";
+import type { CapabilityRef, ModuleManifest, TypeRef } from "@hypit/protocol";
+
+export const standInModuleRef = { name: "@hypit/stand-in", version: "1" } as const;
+export const standInDependency = { module: standInModuleRef } as const;
+
+export const standInTypes = {
+ cardRequest: { module: standInModuleRef, name: "StandInCardRequest" },
+} satisfies Record<string, TypeRef>;
+
+/** Draw one stand-in card. Served by a local media Provider; it draws, it never generates. */
+export const standInCapabilities = {
+ drawCard: { module: standInModuleRef, name: "draw-card" },
+} satisfies Record<string, CapabilityRef>;
+
+export const standInManifest: ModuleManifest = {
+ format: "hypit.module@1",
+ name: standInModuleRef.name,
+ version: standInModuleRef.version,
+ dependencies: [artifactDependency],
+ types: Object.values(standInTypes).map((item) => ({ name: item.name })),
+ capabilities: Object.values(standInCapabilities).map((item) => ({ name: item.name, returns: artifactTypes.blob })),
+ producers: [],
+};
diff --git a/packages/studio/README.md b/packages/studio/README.md
index 04be864c..9ef25f75 100644
--- a/packages/studio/README.md
+++ b/packages/studio/README.md
@@ -83,7 +83,9 @@ that declare `local` pricing, such as the FFmpeg media Endpoint that inspects an
normalizes a media file). Resolution is the same as `plan` and a Build use,
including the Profile's bindings, minus every priced Endpoint; a Need that reaches
a priced Endpoint or none at all stops Studio with the capability named, and the
-author supplies a Candidate for that output or builds the Run.
+author supplies a Candidate for that output or builds the Run. A stand-in card
+(`@hypit/stand-in`, one `<model>-stand-in` Run Fragment per exact model) is such a
+local Need: the picture a generation would have made, drawn from its draft instead.
Studio and an encoded review use the same ordinary Run. Studio evaluates its
free display closure in the browser; building that Run evaluates the
diff --git a/packages/video-cli/test/packages.ts b/packages/video-cli/test/packages.ts
index e6eb38a6..0086c6be 100644
--- a/packages/video-cli/test/packages.ts
+++ b/packages/video-cli/test/packages.ts
@@ -23,6 +23,7 @@ import speech from "../../speech/src/activation.js";
import speechEvidence from "../../speech-evidence/src/activation.js";
import semanticTrack from "../../semantic-track/src/activation.js";
import spatial from "../../spatial/src/activation.js";
+import standIn from "../../stand-in/src/activation.js";
import temporal from "../../temporal/src/activation.js";
import visualIr from "../../visual-ir/src/activation.js";
import svs from "../../svs/src/activation.js";
@@ -59,6 +60,7 @@ export const videoTestPackages = [
bind("@hypit/speech-track", speechTrack),
bind("@hypit/whisperx", whisperX),
bind("@hypit/spatial", spatial),
+ bind("@hypit/stand-in", standIn),
bind("@hypit/temporal", temporal),
bind("@hypit/media-track", mediaTrack),
bind("@hypit/typography-track", typographyTrack),
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9db6e5e1..82d13c72 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -234,6 +234,9 @@ importers:
'@hypit/speech-track-studio':
specifier: workspace:*
version: link:packages/speech-track-studio
+ '@hypit/stand-in':
+ specifier: workspace:*
+ version: link:packages/stand-in
'@hypit/store-sqlite':
specifier: workspace:*
version: link:packages/store-sqlite
@@ -1456,6 +1459,9 @@ importers:
'@hypit/speech':
specifier: workspace:*
version: link:../speech
+ '@hypit/stand-in':
+ specifier: workspace:*
+ version: link:../stand-in
packages/grok-imagine:
dependencies:
@@ -1672,6 +1678,9 @@ importers:
'@hypit/speech':
specifier: workspace:*
version: link:../speech
+ '@hypit/stand-in':
+ specifier: workspace:*
+ version: link:../stand-in
packages/media-pipeline:
dependencies:
@@ -1828,9 +1837,18 @@ importers:
'@hypit/host':
specifier: workspace:*
version: link:../host
+ '@hypit/program-space':
+ specifier: workspace:*
+ version: link:../program-space
'@hypit/protocol':
specifier: workspace:*
version: link:../protocol
+ '@hypit/run':
+ specifier: workspace:*
+ version: link:../run
+ '@hypit/stand-in':
+ specifier: workspace:*
+ version: link:../stand-in
'@hypit/text':
specifier: workspace:*
version: link:../text
@@ -2168,6 +2186,9 @@ importers:
'@hypit/speech':
specifier: workspace:*
version: link:../speech
+ '@hypit/stand-in':
+ specifier: workspace:*
+ version: link:../stand-in
devDependencies:
'@hypit/driver-node':
specifier: workspace:*
@@ -2915,6 +2936,21 @@ importers:
specifier: workspace:*
version: link:../studio-adapter
+ packages/stand-in:
+ dependencies:
+ '@hypit/artifact':
+ specifier: workspace:*
+ version: link:../artifact
+ '@hypit/component-kit':
+ specifier: workspace:*
+ version: link:../component-kit
+ '@hypit/media':
+ specifier: workspace:*
+ version: link:../media
+ '@hypit/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+
packages/store-sqlite:
dependencies:
'@hypit/core':