Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor(stand-in): select generic media candidates from runs

kashorind161dd77c09d
diff --git a/packages/compiler-node/src/run.ts b/packages/compiler-node/src/run.ts
index f6430a3c..b56f977b 100644
--- a/packages/compiler-node/src/run.ts
+++ b/packages/compiler-node/src/run.ts
@@ -221,6 +221,7 @@ export class NodeRunCompiler {
const localRecordIds = new Set(checkedPlan.selections
.filter((selection) => selectedLocalSources.has(selection.candidate))
.map((selection) => selection.record));
+ for (const record of checkedRunGraph.records) localRecordIds.add(record.id);
for (const record of checkedPlan.initialRecords) {
if (localRecordIds.has(record.id)) await admitRecord.call(this.#options.authorCompiler, program, record);
}
diff --git a/packages/core/src/plan.ts b/packages/core/src/plan.ts
index de38df39..b5b8ba3d 100644
--- a/packages/core/src/plan.ts
+++ b/packages/core/src/plan.ts
@@ -133,8 +133,13 @@ function compilePlanning(
graph: CompiledGraph,
request: BuildRequest,
satisfactions: readonly Satisfaction[],
+ runRecords: readonly TypedRecord[],
): PlannedExecution {
- verifyBuildRequest(program, graph, request);
+ const verificationProgram: LinkedProgram = {
+ closure: program.closure,
+ records: [...program.records, ...runRecords],
+ };
+ verifyBuildRequest(verificationProgram, graph, request);
const selectedCandidates = new Map<string, string>();
for (const satisfaction of satisfactions) {
@@ -156,6 +161,14 @@ function compilePlanning(
}
const authored = new Map(program.records.map((record) => [record.id, record]));
+ const availableRunRecords = new Map<string, TypedRecord>();
+ for (const record of runRecords) {
+ invariant(!authored.has(record.id), "RUN_RECORD_CONFLICT", `${record.id} conflicts with authored input`, record.id);
+ invariant(!availableRunRecords.has(record.id), "DUPLICATE_RUN_RECORD", `${record.id} is declared twice`, record.id);
+ availableRunRecords.set(record.id, record);
+ authored.set(record.id, record);
+ }
+ const demandedRunRecords = new Set<string>();
const demandedOperations = new Map<string, OperationNode>();
const resolvedOutputs = new Map<string, ResolvedSource>();
const selections = new Map<string, BuildCandidateSelection>();
@@ -178,6 +191,7 @@ function compilePlanning(
const ref = pending.pop() as GraphValueRef;
if (ref.kind === "record") {
invariant(authored.has(ref.id), "UNKNOWN_RECORD", `unknown authored record ${ref.id}`, ref.id);
+ if (availableRunRecords.has(ref.id)) demandedRunRecords.add(ref.id);
continue;
}
if (ref.kind === "operation-result") {
@@ -267,7 +281,16 @@ function compilePlanning(
record: selection.record,
type: resolveLogicalOutput(graph, selection.output).type,
}));
- const initialRecords = selectedProvidedRecords(program, graph, sortedSelections);
+ const initialRecordsById = new Map<string, TypedRecord>();
+ for (const record of selectedProvidedRecords(program, graph, sortedSelections)) {
+ initialRecordsById.set(record.id, record);
+ }
+ for (const id of demandedRunRecords) {
+ const record = availableRunRecords.get(id)!;
+ invariant(!initialRecordsById.has(id), "RUN_RECORD_CONFLICT", `${id} has multiple sources`, id);
+ initialRecordsById.set(id, record);
+ }
+ const initialRecords = [...initialRecordsById.values()].sort((left, right) => left.id.localeCompare(right.id));
const plan = planContent(steps, goals, outputBindings);
verifyBuildPlan(program, initialRecords, plan);
return { plan, selections: sortedSelections, initialRecords };
@@ -290,7 +313,7 @@ export function planBuild(
format: "hypit.build-request@1",
targets: runGraph.targets,
};
- return compilePlanning(program, graph, request, runGraph.satisfactions);
+ return compilePlanning(program, graph, request, runGraph.satisfactions, runGraph.records);
}
/** Author-only planning convenience. Run compilation should call `planBuild`. */
@@ -299,7 +322,7 @@ export function compileBuild(
graph: CompiledGraph,
request: BuildRequest,
): BuildPlan {
- return compilePlanning(program, graph, request, []).plan;
+ return compilePlanning(program, graph, request, [], []).plan;
}
export function verifyBuildPlan(
diff --git a/packages/core/src/reducer.ts b/packages/core/src/reducer.ts
index 5dbeff46..c3906f73 100644
--- a/packages/core/src/reducer.ts
+++ b/packages/core/src/reducer.ts
@@ -247,6 +247,7 @@ export function start(
): BuildState {
const planned = planBuild(program, graph, {
format: "hypit.run-graph@1",
+ records: [],
candidates: [],
operations: [],
satisfactions: [],
diff --git a/packages/core/test/demand.test.ts b/packages/core/test/demand.test.ts
index 402646fc..1d28f8a4 100644
--- a/packages/core/test/demand.test.ts
+++ b/packages/core/test/demand.test.ts
@@ -293,6 +293,7 @@ function startSelected(
): BuildState {
const planned = planBuild(program, graph, {
format: "hypit.run-graph@1",
+ records: [],
candidates: [],
operations: [],
satisfactions,
diff --git a/packages/gpt-image/package.json b/packages/gpt-image/package.json
index 56af38ef..241a5eed 100644
--- a/packages/gpt-image/package.json
+++ b/packages/gpt-image/package.json
@@ -29,7 +29,6 @@
"@hypit/run": "workspace:*",
"@hypit/seedance": "workspace:*",
"@hypit/spatial": "workspace:*",
- "@hypit/speech": "workspace:*",
- "@hypit/stand-in": "workspace:*"
+ "@hypit/speech": "workspace:*"
}
}
diff --git a/packages/gpt-image/src/activation.ts b/packages/gpt-image/src/activation.ts
index ed4bf252..1c2b9921 100644
--- a/packages/gpt-image/src/activation.ts
+++ b/packages/gpt-image/src/activation.ts
@@ -24,7 +24,6 @@ export const hypitPackage = {
components: [gptImageComponent],
hostFacets: [
gptImageDefinition.hostFacet,
- ...(gptImageDefinition.runFragmentFacet === undefined ? [] : [gptImageDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: gptImageModuleRef,
declaration: gptImageMarkupSurfaces.find((item) => item.name === "image")!,
diff --git a/packages/grok-imagine/src/activation.ts b/packages/grok-imagine/src/activation.ts
index 20cce3e8..dddbd140 100644
--- a/packages/grok-imagine/src/activation.ts
+++ b/packages/grok-imagine/src/activation.ts
@@ -14,7 +14,6 @@ export const hypitPackage = {
components: [grokImagineComponent],
hostFacets: [
grokImagineDefinition.hostFacet,
- ...(grokImagineDefinition.runFragmentFacet === undefined ? [] : [grokImagineDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: grokImagineModuleRef,
declaration: grokImagineMarkupSurfaces.find((item) => item.name === "video")!,
diff --git a/packages/media-execution/src/card.ts b/packages/media-execution/src/card.ts
index 60b9091f..d3fb067e 100644
--- a/packages/media-execution/src/card.ts
+++ b/packages/media-execution/src/card.ts
@@ -9,11 +9,11 @@ import { verifyStandInCardRequest } from "@hypit/stand-in";
import type { StandInCardRequest } from "@hypit/stand-in";
/**
- * A stand-in card, drawn from the request the model would have received.
+ * A generic stand-in card, drawn from only the media shape a Run supplies.
*
* The card is pixels, not a model call: a flat ground coloured by kind, a thick border, a diagonal
- * STAND-IN watermark, the kind in large type, the model, the frame and duration, an excerpt of the
- * prompt, and for video a running timecode and progress bar. Everything is drawn here with a
+ * STAND-IN watermark, the kind in large type, the frame and duration, and for video a running
+ * timecode and progress bar. Everything is drawn here with a
* built-in 5x7 bitmap font so the same card comes out of every machine, with or without a font
* library in its ffmpeg. The PNG is deterministic for a given request.
*/
@@ -215,27 +215,6 @@ function textWidth(value: string, scale: number): number {
return Math.max(0, [...value].length * ADVANCE * scale - scale);
}
-/** Uppercase, one line per row, at most `maxLines` rows of `maxChars`; the last row ends in an ellipsis. */
-function wrap(value: string, maxChars: number, maxLines: number): readonly string[] {
- const words = value.replace(/\s+/gu, " ").trim().split(" ").filter((word) => word.length > 0);
- const lines: string[] = [];
- let current = "";
- for (const word of words) {
- const candidate = current.length === 0 ? word : `${current} ${word}`;
- if (candidate.length <= maxChars) { current = candidate; continue; }
- if (current.length > 0) lines.push(current);
- current = word.length > maxChars ? word.slice(0, maxChars) : word;
- if (lines.length === maxLines) break;
- }
- if (current.length > 0 && lines.length < maxLines) lines.push(current);
- const truncated = lines.join(" ").length < words.join(" ").length;
- if (truncated && lines.length > 0) {
- const last = lines[lines.length - 1]!;
- lines[lines.length - 1] = `${last.slice(0, Math.max(0, maxChars - 3))}...`;
- }
- return lines;
-}
-
function seconds(video: NonNullable<StandInCardRequest["video"]>): number {
return video.frameCount * video.frameRate.denominator / video.frameRate.numerator;
}
@@ -294,18 +273,15 @@ export function drawStandInCard(request: StandInCardRequest): Uint8Array {
const kindTag = request.kind === "video" ? "NOT YET GENERATED" : "NOT YET GENERATED";
raster.text(kindTag, width - margin - textWidth(kindTag, tagScale), margin, tagScale, MUTED);
- // Centre block: kind, model, frame and duration, prompt excerpt.
+ // Centre block: only the generic media shape selected by the Run.
const titleScale = Math.max(4, Math.floor(m / 70));
const lineScale = Math.max(2, Math.floor(m / 200));
- const promptScale = Math.max(2, Math.floor(m / 240));
const title = request.kind.toUpperCase();
const frameLine = request.video === undefined
? `${width} X ${height}`
: `${width} X ${height} ${seconds(request.video).toFixed(2)} S ${(request.video.frameRate.numerator / request.video.frameRate.denominator).toFixed(2)} FPS`;
- const promptLines = wrap(request.prompt, Math.max(8, Math.floor((width - margin * 2) / (ADVANCE * promptScale))), 4);
const blockHeight = GLYPH_HEIGHT * titleScale + lineScale * 6
- + GLYPH_HEIGHT * lineScale * 2 + lineScale * 4
- + (promptLines.length === 0 ? 0 : lineScale * 8 + promptLines.length * (GLYPH_HEIGHT * promptScale + promptScale * 3));
+ + GLYPH_HEIGHT * lineScale;
let y = Math.round((height - band - blockHeight) / 2);
const centred = (value: string, scale: number, color: Rgb): void => {
raster.text(value, Math.round((width - textWidth(value, scale)) / 2), y, scale, color);
@@ -313,19 +289,32 @@ export function drawStandInCard(request: StandInCardRequest): Uint8Array {
};
centred(title, titleScale, TEXT);
y += lineScale * 6;
- centred(request.model, lineScale, ACCENT);
- y += lineScale * 4;
centred(frameLine, lineScale, MUTED);
- if (promptLines.length > 0) {
- y += lineScale * 8;
- for (const line of promptLines) {
- centred(line, promptScale, TEXT);
- y += promptScale * 3;
- }
- }
return raster.png();
}
+export function drawStandInSilence(sampleFrames: number): Uint8Array {
+ if (!Number.isSafeInteger(sampleFrames) || sampleFrames < 1) {
+ throw new Error("Stand-in silence sampleFrames must be a positive integer");
+ }
+ const dataBytes = sampleFrames * 2 * 2;
+ const wav = Buffer.alloc(44 + dataBytes);
+ wav.write("RIFF", 0);
+ wav.writeUInt32LE(36 + dataBytes, 4);
+ wav.write("WAVE", 8);
+ wav.write("fmt ", 12);
+ wav.writeUInt32LE(16, 16);
+ wav.writeUInt16LE(1, 20);
+ wav.writeUInt16LE(2, 22);
+ wav.writeUInt32LE(48_000, 24);
+ wav.writeUInt32LE(48_000 * 4, 28);
+ wav.writeUInt16LE(4, 32);
+ wav.writeUInt16LE(16, 34);
+ wav.write("data", 36);
+ wav.writeUInt32LE(dataBytes, 40);
+ return Uint8Array.from(wav);
+}
+
/** The band a video card carries under its picture: timecode, frame counter and progress bar. */
export function drawStandInBand(request: StandInCardRequest, frame: number): Uint8Array {
verifyStandInCardRequest(request);
diff --git a/packages/media-execution/src/execute.ts b/packages/media-execution/src/execute.ts
index a51fd08a..6d55d641 100644
--- a/packages/media-execution/src/execute.ts
+++ b/packages/media-execution/src/execute.ts
@@ -32,7 +32,8 @@ import {
} from "@hypit/protocol";
import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/protocol";
-import { renderStandInCard, standInCardNeed } from "./card.js";
+import { drawStandInSilence, renderStandInCard, standInCardNeed } from "./card.js";
+import { verifyStandInSilenceRequest } from "@hypit/stand-in";
import { parseMediaInspection } from "./probe.js";
import {
compositeAnimatedWebpFrame,
@@ -858,6 +859,15 @@ export async function executeDrawStandInCard(
return artifactResult(value as unknown as BlobRef);
}
+/** Materialize the generic silent-audio Candidate selected by a Run. */
+export async function executeDrawStandInSilence(
+ env: MediaExecutionEnvironment,
+ constraints: CanonicalValue,
+): Promise<MediaOperationResult> {
+ verifyStandInSilenceRequest(constraints);
+ return artifactResult(await env.artifacts.put(drawStandInSilence(constraints.sampleFrames), "audio/wav"));
+}
+
type TransformPlan = {
readonly videoFilters: readonly string[];
readonly audioFilters: readonly string[];
diff --git a/packages/media-execution/src/index.ts b/packages/media-execution/src/index.ts
index 6dbe5f05..04bd8007 100644
--- a/packages/media-execution/src/index.ts
+++ b/packages/media-execution/src/index.ts
@@ -8,9 +8,10 @@ export {
executeRenderTimelineAudio,
executeRenderStillVideo,
executeDrawStandInCard,
+ executeDrawStandInSilence,
executeTransformMedia,
} from "./execute.js";
-export { drawStandInBand, drawStandInCard } from "./card.js";
+export { drawStandInBand, drawStandInCard, drawStandInSilence } from "./card.js";
export { parseMediaInspection } from "./probe.js";
export { verifyCompositableSurfaceBytes } from "./surface.js";
export { probeMediaToolchain } from "./toolchain.js";
diff --git a/packages/media-track/preview/build.svrun b/packages/media-track/preview/build.svrun
index 35fd619c..42e2994f 100644
--- a/packages/media-track/preview/build.svrun
+++ b/packages/media-track/preview/build.svrun
@@ -2,5 +2,24 @@
<svrun version="1">
<author source="./preview.svml"/>
+ <import from="@hypit/stand-in@1" as="stand-in"/>
<target output="final.video"/>
+ <fragment id="take-card" using="stand-in:video">
+ <input name="canvas" from="vertical"/>
+ <input name="duration" value="8"/>
+ <input name="clock" from="clock"/>
+ </fragment>
+ <fragment id="motion-a-card" using="stand-in:video">
+ <input name="canvas" from="vertical"/>
+ <input name="duration" value="5"/>
+ <input name="clock" from="clock"/>
+ </fragment>
+ <fragment id="motion-b-card" using="stand-in:video">
+ <input name="canvas" from="vertical"/>
+ <input name="duration" value="5"/>
+ <input name="clock" from="clock"/>
+ </fragment>
+ <satisfy output="take.video" candidate="take-card.video"/>
+ <satisfy output="motion-a.video" candidate="motion-a-card.video"/>
+ <satisfy output="motion-b.video" candidate="motion-b-card.video"/>
</svrun>
diff --git a/packages/media-track/preview/preview.svml b/packages/media-track/preview/preview.svml
index 3fd79c7e..6926937b 100644
--- a/packages/media-track/preview/preview.svml
+++ b/packages/media-track/preview/preview.svml
@@ -10,12 +10,13 @@
preserve how it was authored; regenerating the encoded frame is a normal Build.
A placeholder shows the Frame treatment and nothing inside it. To look at the component before
- buying its pictures, stand in for each generation in the Run; the card is derived from the
- model's own draft, and the Profile's local media Endpoint normalizes it, in Studio as in a Build:
+ buying its pictures, select a generic stand-in Candidate in the Run. The Run supplies only the
+ media shape its composition needs; the Profile's local media Endpoint draws and normalizes it,
+ in Studio as in a Build:
- <import from="@hypit/seedance@1" as="seedance"/>
- <fragment id="a-card" using="seedance:mini-stand-in">
- <input name="draft" from="motion-a.draft"/><input name="prompt" from="direction"/><input name="clock" from="clock"/>
+ <import from="@hypit/stand-in@1" as="stand-in"/>
+ <fragment id="a-card" using="stand-in:video">
+ <input name="canvas" from="vertical"/><input name="duration" value="5"/><input name="clock" from="clock"/>
</fragment>
<satisfy output="motion-a.video" candidate="a-card.video"/>
diff --git a/packages/minimax-h3/src/activation.ts b/packages/minimax-h3/src/activation.ts
index e253c792..7229333a 100644
--- a/packages/minimax-h3/src/activation.ts
+++ b/packages/minimax-h3/src/activation.ts
@@ -4,7 +4,6 @@ import { minimaxH3Component, minimaxH3Definition, minimaxH3Manifest, minimaxH3Mo
import { decodeMinimaxFrameVideoSurface, decodeMinimaxReferenceVideoSurface, decodeMinimaxTextVideoSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: minimaxH3Manifest }], components: [minimaxH3Component], hostFacets: [
minimaxH3Definition.hostFacet,
- ...(minimaxH3Definition.runFragmentFacet === undefined ? [] : [minimaxH3Definition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,
declaration: minimaxH3MarkupSurfaces.find((item) => item.name === "text-video")!, handler: decodeMinimaxTextVideoSurface }),
createMarkupSurfaceHostFacet({ module: minimaxH3ModuleRef,
diff --git a/packages/model-kit/package.json b/packages/model-kit/package.json
index 3132e021..f1bae183 100644
--- a/packages/model-kit/package.json
+++ b/packages/model-kit/package.json
@@ -13,10 +13,7 @@
"@hypit/elaborator": "workspace:*",
"@hypit/generation": "workspace:*",
"@hypit/host": "workspace:*",
- "@hypit/program-space": "workspace:*",
"@hypit/protocol": "workspace:*",
- "@hypit/run": "workspace:*",
- "@hypit/stand-in": "workspace:*",
"@hypit/text": "workspace:*"
}
}
diff --git a/packages/model-kit/src/index.ts b/packages/model-kit/src/index.ts
index 395adff5..64018f9d 100644
--- a/packages/model-kit/src/index.ts
+++ b/packages/model-kit/src/index.ts
@@ -27,15 +27,7 @@ import type {
GenerationPortTable,
GenerationRequestDraft,
} from "@hypit/generation";
-import { programSpaceDependency } from "@hypit/program-space";
-import type { ProgramClock } from "@hypit/program-space";
-import { createRunFragmentHostFacet } from "@hypit/run";
-import type { RunFragmentHostFacet } from "@hypit/run";
-import { standInDependency } from "@hypit/stand-in";
import { textDependency, textTypes } from "@hypit/text";
-
-import { exactModelStandIn, standInCardRequestFromDraft, standInNeed } from "./stand-in.js";
-import type { ExactModelStandIn } from "./stand-in.js";
import type { Text } from "@hypit/text";
import {
canonicalize,
@@ -95,8 +87,6 @@ export type ExactModelEndpoint = {
readonly textBindings: Readonly<Record<string, ExactModelTextBindingEndpoint>>;
readonly ports: GenerationPortTable;
readonly fragment: ReturnType<typeof sealGraphFragment>;
- /** The drawn stand-in for this model's output; absent for a model whose output is not a picture. */
- readonly standIn?: ExactModelStandIn;
};
export type ExactModelMediaBindingEndpoint = {
@@ -135,8 +125,6 @@ export type ExactModelModule<Key extends string = string> = {
};
/** Inert declaration of these exact models for Hosts that address one directly. */
readonly hostFacet: ExactModelHostFacet;
- /** Run Fragments that stand in for this module's picture outputs, one per model, named `<key>-stand-in`. */
- readonly runFragmentFacet?: RunFragmentHostFacet;
};
export const exactModelHostAbi = "hypit.exact-model-host@1";
@@ -384,11 +372,7 @@ export function defineExactModelModule<const Key extends string>(
assert(new Set(keys).size === keys.length, `${options.module.name} repeats an endpoint key`);
const models = options.endpoints.map((item) => item.ports.model);
assert(new Set(models).size === models.length, `${options.module.name} repeats an exact model`);
- const endpointData = options.endpoints.map((spec) => {
- const refs = endpointRef(options.module, spec);
- const standIn = exactModelStandIn({ module: options.module, producerName: spec.producerName, draftType: refs.draftType, ports: spec.ports });
- return { spec, ...refs, ...(standIn === undefined ? {} : { standIn }) };
- });
+ const endpointData = options.endpoints.map((spec) => ({ spec, ...endpointRef(options.module, spec) }));
const manifest: ModuleManifest = {
format: "hypit.module@1",
@@ -398,8 +382,6 @@ export function defineExactModelModule<const Key extends string>(
{ module: generationModuleRef },
artifactDependency,
textDependency,
- standInDependency,
- programSpaceDependency,
],
types: endpointData.flatMap((item) => [
{
@@ -455,12 +437,6 @@ export function defineExactModelModule<const Key extends string>(
outputs: [{ name: "request", type: item.requestType }],
needs: [],
},
- ...(item.standIn === undefined ? [] : [{
- name: item.standIn.producer.name,
- inputs: item.standIn.inputs,
- outputs: [],
- needs: [standInNeed],
- }]),
]),
};
@@ -491,13 +467,9 @@ export function defineExactModelModule<const Key extends string>(
textBindings: item.textBindings,
ports: item.spec.ports,
fragment,
- ...(item.standIn === undefined ? {} : { standIn: item.standIn }),
} satisfies ExactModelEndpoint];
}));
- const standIns = endpointData.flatMap((item) =>
- item.standIn === undefined ? [] : [[`${item.spec.key}-stand-in`, item.standIn.fragment] as const]);
-
return {
module: { ...options.module },
manifest,
@@ -579,24 +551,8 @@ export function defineExactModelModule<const Key extends string>(
needs: {},
}),
},
- ...(item.standIn === undefined ? [] : [{
- producer: item.standIn.producer,
- handler: ({ inputs }: ProducerHandlerContext) => {
- const draft = inlineValue<GenerationRequestDraft>(inputs.draft!.value, `${item.spec.key} draft`);
- const prompt = inlineValue<Text>(inputs.prompt!.value, `${item.spec.key} stand-in prompt`);
- const clock = inputs.clock === undefined ? undefined : inlineValue<ProgramClock>(inputs.clock.value, `${item.spec.key} stand-in clock`);
- const request = standInCardRequestFromDraft({ ports: item.spec.ports, draft, prompt, ...(clock === undefined ? {} : { clock }) });
- return { outputs: {}, needs: { card: canonicalize(request as unknown as CanonicalValue) } };
- },
- }]),
]),
},
- ...(standIns.length === 0 ? {} : {
- runFragmentFacet: createRunFragmentHostFacet({
- name: `${options.module.name}@${options.module.version}`,
- fragments: Object.fromEntries(standIns),
- }),
- }),
};
}
@@ -708,5 +664,3 @@ export function createExactModelPrimaryGenerationFragment(
}],
});
}
-export { standInCardRequestFromDraft, standInFrameFromDraft } from "./stand-in.js";
-export type { ExactModelStandIn } from "./stand-in.js";
diff --git a/packages/model-kit/src/stand-in.ts b/packages/model-kit/src/stand-in.ts
deleted file mode 100644
index c7f60585..00000000
--- a/packages/model-kit/src/stand-in.ts
+++ /dev/null
@@ -1,139 +0,0 @@
-import { artifactTypes } from "@hypit/artifact";
-import { sealGraphFragment } from "@hypit/elaborator";
-import type { GenerationPortTable, GenerationRequestDraft } from "@hypit/generation";
-import { assertProgramClockIdentity, programSpaceTypes } from "@hypit/program-space";
-import type { ProgramClock } from "@hypit/program-space";
-import type { ModuleRef, ProducerRef, TypeRef } from "@hypit/protocol";
-import { sealStandInCardRequest, standInCapabilities } from "@hypit/stand-in";
-import type { StandInCardRequest } from "@hypit/stand-in";
-import { textTypes } from "@hypit/text";
-import type { Text } from "@hypit/text";
-
-/**
- * A stand-in for an exact model's output is derived from that model's own request draft: the
- * frame from its resolution and aspect ports, the duration from its duration port, the model from
- * its port table, the prompt from the same Text edge the model would read. Nothing is restated by
- * hand, so the card follows the Source. What cannot be derived is refused with the reason.
- */
-
-function assert(condition: unknown, message: string): asserts condition {
- if (!condition) throw new Error(message);
-}
-
-/** How many pixels a resolution token names, and on which side. */
-const RESOLUTIONS: Readonly<Record<string, { readonly pixels: number; readonly side: "shorter" | "longer" }>> = {
- "480p": { pixels: 480, side: "shorter" },
- "720p": { pixels: 720, side: "shorter" },
- "1080p": { pixels: 1080, side: "shorter" },
- "4k": { pixels: 2160, side: "shorter" },
- "1K": { pixels: 1024, side: "longer" },
- "2K": { pixels: 2048, side: "longer" },
- "4K": { pixels: 4096, side: "longer" },
-};
-
-function single(draft: GenerationRequestDraft, port: string): string | number | boolean | undefined {
- const values = draft.ports[port];
- if (values === undefined || values.length === 0) return undefined;
- const value = values[0];
- return typeof value === "object" ? undefined : value;
-}
-
-function even(value: number): number {
- const rounded = Math.round(value);
- return rounded % 2 === 0 ? rounded : rounded + 1;
-}
-
-export function standInFrameFromDraft(ports: GenerationPortTable, draft: GenerationRequestDraft): { readonly width: number; readonly height: number } {
- const size = single(draft, "size");
- if (typeof size === "string") {
- const match = /^(\d+)\s*[xX×]\s*(\d+)$/u.exec(size.trim());
- assert(match !== null, `${ports.model} size port "${size}" is not WIDTHxHEIGHT; no stand-in frame can be derived`);
- return { width: even(Number(match[1])), height: even(Number(match[2])) };
- }
- const resolution = single(draft, "resolution");
- assert(typeof resolution === "string" && RESOLUTIONS[resolution] !== undefined,
- `${ports.model} names no resolution a stand-in understands (${String(resolution)}); the card cannot know its frame`);
- const aspect = single(draft, "aspectRatio");
- let ratio: { readonly w: number; readonly h: number };
- if (typeof aspect === "string") {
- const match = /^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/u.exec(aspect.trim());
- assert(match !== null, `${ports.model} aspectRatio "${aspect}" is not W:H; no stand-in frame can be derived`);
- ratio = { w: Number(match[1]), h: Number(match[2]) };
- } else {
- assert(ports.result === "image", `${ports.model} declares no aspect ratio; a video stand-in cannot guess its frame`);
- ratio = { w: 1, h: 1 };
- }
- const { pixels, side } = RESOLUTIONS[resolution]!;
- const portrait = ratio.h > ratio.w;
- const fixShorter = side === "shorter";
- // The named pixel count lands on the shorter side (480p, 720p…) or the longer side (1K, 2K…).
- const widthIsFixed = fixShorter ? portrait : !portrait;
- return widthIsFixed
- ? { width: even(pixels), height: even(pixels * ratio.h / ratio.w) }
- : { width: even(pixels * ratio.w / ratio.h), height: even(pixels) };
-}
-
-export function standInCardRequestFromDraft(args: {
- readonly ports: GenerationPortTable;
- readonly draft: GenerationRequestDraft;
- readonly prompt: Text;
- readonly clock?: ProgramClock;
-}): StandInCardRequest {
- const { ports, draft } = args;
- assert(ports.result !== "audio", `${ports.model} produces audio; a stand-in card is a picture`);
- const frame = standInFrameFromDraft(ports, draft);
- const base = { ...frame, model: ports.model, prompt: args.prompt.value };
- if (ports.result === "image") return sealStandInCardRequest({ kind: "image", ...base });
- const clock = args.clock;
- assert(clock !== undefined, `a video stand-in for ${ports.model} needs the piece's clock`);
- assertProgramClockIdentity(clock);
- const duration = single(draft, "duration");
- assert(typeof duration === "number" && Number.isFinite(duration) && duration > 0,
- `${ports.model} has no fixed duration (${String(duration)}); write the seconds, an automatic duration has no stand-in`);
- const frameCount = Math.round(duration * clock.frameRate.numerator / clock.frameRate.denominator);
- assert(Number.isSafeInteger(frameCount) && frameCount > 0, `${ports.model} duration yields no frames on this clock`);
- const audio = single(draft, "generateAudio") === true;
- return sealStandInCardRequest({ kind: "video", ...base, video: { frameRate: clock.frameRate, frameCount, ...(audio ? { audio: "silence" as const } : {}) } });
-}
-
-export type ExactModelStandIn = {
- readonly producer: ProducerRef;
- readonly fragment: ReturnType<typeof sealGraphFragment>;
- /** The export name a Run satisfies with: `image` or `video`, the Blob the model would have made. */
- readonly output: "image" | "video";
- readonly inputs: readonly { readonly name: string; readonly type: TypeRef }[];
-};
-
-/** The stand-in shape for one endpoint, or none for a model whose output is not a picture. */
-export function exactModelStandIn(args: {
- readonly module: ModuleRef;
- readonly producerName: string;
- readonly draftType: TypeRef;
- readonly ports: GenerationPortTable;
-}): ExactModelStandIn | undefined {
- if (args.ports.result === "audio") return undefined;
- const video = args.ports.result === "video";
- const producer = { module: args.module, name: `stand-in-${args.producerName}` };
- const inputs = [
- { name: "draft", type: args.draftType },
- { name: "prompt", type: textTypes.text },
- ...(video ? [{ name: "clock", type: programSpaceTypes.clock }] : []),
- ];
- const fragment = sealGraphFragment({
- inputs,
- operations: [{
- id: "card",
- producer,
- inputs: Object.fromEntries(inputs.map((input) => [input.name, { kind: "fragment-input" as const, name: input.name }])),
- result: { kind: "need", name: "card" },
- }],
- exports: [{
- name: video ? "video" : "image",
- type: artifactTypes.blob,
- root: { kind: "fragment-operation", operation: "card" },
- }],
- });
- return { producer, fragment, output: video ? "video" : "image", inputs };
-}
-
-export const standInNeed = { name: "card", capability: standInCapabilities.drawCard, returns: artifactTypes.blob } as const;
diff --git a/packages/model-kit/test/stand-in.test.ts b/packages/model-kit/test/stand-in.test.ts
deleted file mode 100644
index 7242dcff..00000000
--- a/packages/model-kit/test/stand-in.test.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-
-import { sealGenerationRequestDraft } from "@hypit/generation";
-import type { GenerationPortTable } from "@hypit/generation";
-import { sealText } from "@hypit/text";
-
-import { defineExactModelModule, standInCardRequestFromDraft, standInFrameFromDraft } from "@hypit/model-kit";
-
-const clip: GenerationPortTable = {
- model: "clip-model",
- result: "video",
- ports: [
- { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },
- { name: "resolution", value: { kind: "enum", values: ["480p", "720p", "1080p"] }, minItems: 1, maxItems: 1 },
- { name: "aspectRatio", value: { kind: "enum", values: ["9:16", "16:9", "1:1"] }, minItems: 1, maxItems: 1 },
- { name: "duration", value: { kind: "enum", values: [-1, 4, 5, 8] }, minItems: 1, maxItems: 1 },
- { name: "generateAudio", value: { kind: "boolean" }, minItems: 0, maxItems: 1 },
- ],
- requires: [],
-};
-const picture: GenerationPortTable = {
- model: "picture-model",
- result: "image",
- ports: [
- { name: "prompt", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 },
- { name: "resolution", value: { kind: "enum", values: ["1K", "2K"] }, minItems: 1, maxItems: 1 },
- { name: "aspectRatio", value: { kind: "enum", values: ["3:2", "1:1"] }, minItems: 0, maxItems: 1 },
- ],
- requires: [],
-};
-const voice: GenerationPortTable = {
- model: "voice-model",
- result: "audio",
- ports: [{ name: "text", value: { kind: "text", maxChars: 1000 }, minItems: 1, maxItems: 1 }],
- requires: [],
-};
-const clock = { frameRate: { numerator: 30, denominator: 1 } };
-
-test("a stand-in card is derived from the model's draft: frame, duration, model and prompt", () => {
- const draft = sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] });
- const request = standInCardRequestFromDraft({ ports: clip, draft, prompt: sealText("A quiet kitchen"), clock });
- assert.deepEqual(request, {
- kind: "video", width: 720, height: 1280, model: "clip-model", prompt: "A quiet kitchen",
- video: { frameRate: { numerator: 30, denominator: 1 }, frameCount: 150 },
- });
- const spoken = standInCardRequestFromDraft({ ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5], generateAudio: [true] }), prompt: sealText("x"), clock });
- assert.equal(spoken.video?.audio, "silence", "a model that would have spoken leaves a silent track to cut against");
- assert.deepEqual(standInFrameFromDraft(clip, sealGenerationRequestDraft(clip, { resolution: ["1080p"], aspectRatio: ["16:9"], duration: [4] })),
- { width: 1920, height: 1080 });
- assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["1K"] })), { width: 1024, height: 1024 });
- assert.deepEqual(standInFrameFromDraft(picture, sealGenerationRequestDraft(picture, { resolution: ["2K"], aspectRatio: ["3:2"] })), { width: 2048, height: 1366 });
-});
-
-test("what a stand-in cannot derive it refuses with the reason", () => {
- assert.throws(() => standInCardRequestFromDraft({
- ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [-1] }),
- prompt: sealText("x"), clock,
- }), /automatic duration has no stand-in/u);
- assert.throws(() => standInCardRequestFromDraft({
- ports: clip, draft: sealGenerationRequestDraft(clip, { resolution: ["720p"], aspectRatio: ["9:16"], duration: [5] }), prompt: sealText("x"),
- }), /needs the piece's clock/u);
- assert.throws(() => standInCardRequestFromDraft({
- ports: voice, draft: sealGenerationRequestDraft(voice, {}), prompt: sealText("x"),
- }), /produces audio/u);
-});
-
-test("an exact model module exports one stand-in Run Fragment per picture model and none for audio", () => {
- const module = defineExactModelModule({
- module: { name: "@example/models", version: "1" },
- endpoints: [
- { key: "clip", requestTypeName: "ClipRequest", producerName: "request-clip", ports: clip },
- { key: "picture", requestTypeName: "PictureRequest", producerName: "request-picture", ports: picture },
- { key: "voice", requestTypeName: "VoiceRequest", producerName: "request-voice", ports: voice },
- ],
- });
- assert.deepEqual(Object.keys(module.runFragmentFacet!.implementation.fragments).sort(), ["clip-stand-in", "picture-stand-in"]);
- assert.equal(module.runFragmentFacet!.offers[0], "@example/models@1");
- assert.deepEqual(module.endpoints.clip.standIn!.inputs.map((input) => input.name), ["draft", "prompt", "clock"]);
- assert.deepEqual(module.endpoints.picture.standIn!.inputs.map((input) => input.name), ["draft", "prompt"]);
- assert.equal(module.endpoints.voice.standIn, undefined);
- assert.ok(module.manifest.producers.some((item) => item.name === "stand-in-request-clip"
- && item.needs[0]?.capability.name === "draw-card"));
- assert.ok(module.manifest.dependencies.some((item) => item.module.name === "@hypit/stand-in"));
-});
diff --git a/packages/nano-banana/src/activation.ts b/packages/nano-banana/src/activation.ts
index f8bd921c..ff2c6ea4 100644
--- a/packages/nano-banana/src/activation.ts
+++ b/packages/nano-banana/src/activation.ts
@@ -6,7 +6,6 @@ import {
import { decodeNanoBananaImageSurface, decodeNanoBananaProImageSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: nanoBananaManifest }], components: [nanoBananaComponent], hostFacets: [
nanoBananaDefinition.hostFacet,
- ...(nanoBananaDefinition.runFragmentFacet === undefined ? [] : [nanoBananaDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,
declaration: nanoBananaMarkupSurfaces.find((item) => item.name === "image")!, handler: decodeNanoBananaImageSurface }),
createMarkupSurfaceHostFacet({ module: nanoBananaModuleRef,
diff --git a/packages/protocol/src/build.ts b/packages/protocol/src/build.ts
index 11f1706f..b6a8eb6f 100644
--- a/packages/protocol/src/build.ts
+++ b/packages/protocol/src/build.ts
@@ -148,6 +148,8 @@ export type BuildPlan = {
/** The complete execution intent compiled from one `.svrun` source. */
export type RunGraph = {
readonly format: "hypit.run-graph@1";
+ /** Literal values authored by this Run and reached only through its selected Fragments. */
+ readonly records: readonly TypedRecord[];
readonly candidates: readonly Candidate[];
readonly operations: readonly OperationNode[];
readonly satisfactions: readonly Satisfaction[];
diff --git a/packages/provider-media-local/src/provider.ts b/packages/provider-media-local/src/provider.ts
index b231a521..5f322565 100644
--- a/packages/provider-media-local/src/provider.ts
+++ b/packages/provider-media-local/src/provider.ts
@@ -10,6 +10,7 @@ import {
executeRenderTimelineAudio,
executeRenderStillVideo,
executeDrawStandInCard,
+ executeDrawStandInSilence,
executeTransformMedia,
} from "@hypit/media-execution";
import type { MediaExecutionEnvironment, MediaOperationResult } from "@hypit/media-execution";
@@ -127,6 +128,12 @@ export function createLocalMediaProvider(config: CreateLocalMediaProviderOptions
returns: artifactTypes.blob,
handler: operation(executeDrawStandInCard),
},
+ {
+ lifecycle: "immediate" as const,
+ capability: standInCapabilities.drawSilence,
+ returns: artifactTypes.blob,
+ handler: operation(executeDrawStandInSilence),
+ },
{
lifecycle: "immediate" as const,
capability: mediaPipelineCapabilities.projectSpeechEvidenceAudio,
diff --git a/packages/provider-media-local/test/provider.test.ts b/packages/provider-media-local/test/provider.test.ts
index 3c8c0c35..7647e287 100644
--- a/packages/provider-media-local/test/provider.test.ts
+++ b/packages/provider-media-local/test/provider.test.ts
@@ -871,7 +871,7 @@ test("local media Provider draws a stand-in card as a picture and as an exact si
return result.value as Awaited<ReturnType<MemoryResourceStore["put"]>>;
};
const picture = await fulfillBlob(need("need:card-image", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
- kind: "image", width: 640, height: 400, model: "picture-model", prompt: "A quiet kitchen counter with fresh basil",
+ kind: "image", width: 640, height: 400,
})));
assert.equal(picture.kind, "blob");
assert.equal(picture.mediaType, "image/png");
@@ -880,25 +880,31 @@ test("local media Provider draws a stand-in card as a picture and as an exact si
assert.deepEqual(pngSize(bytes), { width: 640, height: 400 });
const clip = await fulfillBlob(need("need:card-video", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
- kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Locked medium close-up",
+ kind: "video", width: 320, height: 568,
video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12 },
})));
assert.equal(clip.mediaType, "video/mp4");
const inspection = await inspectArtifact(resources, clip);
const visual = inspection.streams.find((item) => item.kind === "video");
assert.ok(visual !== undefined && visual.kind === "video");
- assert.equal(inspection.streams.length, 1, "a stand-in clip carries no sound unless the model would have");
+ assert.equal(inspection.streams.length, 1, "the card request alone decides whether a clip carries silence");
assert.equal(visual.width, 320);
assert.equal(visual.height, 568);
assert.equal(visual.decodedUnitCount, 12);
const spoken = await fulfillBlob(need("need:card-spoken", standInCapabilities.drawCard, artifactTypes.blob, canonicalize({
- kind: "video", width: 320, height: 568, model: "clip-model", prompt: "Spoken dialogue",
+ kind: "video", width: 320, height: 568,
video: { frameRate: { numerator: 24, denominator: 1 }, frameCount: 12, audio: "silence" },
})));
const spokenStreams = (await inspectArtifact(resources, spoken)).streams;
const sound = spokenStreams.find((item) => item.kind === "audio");
- assert.ok(sound !== undefined && sound.kind === "audio", "a model that would have spoken leaves a silent track");
+ assert.ok(sound !== undefined && sound.kind === "audio", "a generic video stand-in can carry a silent track");
assert.equal(sound.sampleRate, 48_000);
assert.equal(spokenStreams.length, 2);
+
+ const silence = await fulfillBlob(need("need:stand-in-silence", standInCapabilities.drawSilence, artifactTypes.blob,
+ canonicalize({ sampleRate: 48_000, channels: 2, sampleFrames: 48_000 })));
+ assert.equal(silence.mediaType, "audio/wav");
+ const silenceBytes = await resources.get(silence.resource);
+ assert.equal(silenceBytes?.byteLength, 44 + 48_000 * 4);
});
diff --git a/packages/run-markup/src/syntax.ts b/packages/run-markup/src/syntax.ts
index ab12ddd3..8aa5a389 100644
--- a/packages/run-markup/src/syntax.ts
+++ b/packages/run-markup/src/syntax.ts
@@ -1,4 +1,4 @@
-import type { TypeRef } from "@hypit/protocol";
+import type { CanonicalValue, TypeRef } from "@hypit/protocol";
import type {
RunBuildRecord,
RunCandidateDeclaration,
@@ -157,6 +157,17 @@ function buildRecord(element: StructuredElement): RunBuildRecord {
};
}
+function scalar(value: string): CanonicalValue {
+ if (value === "true") return true;
+ if (value === "false") return false;
+ if (value === "null") return null;
+ if (/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/u.test(value)) {
+ const number = Number(value);
+ if (Number.isFinite(number)) return number;
+ }
+ return value;
+}
+
function fragment(element: StructuredElement): RunFragmentInstance {
exactAttributes(element, ["id", "using"]);
const using = stringAttribute(element, "using")!;
@@ -164,13 +175,19 @@ function fragment(element: StructuredElement): RunFragmentInstance {
if (separator <= 0 || separator === using.length - 1) {
fail(element, "RUN_FRAGMENT_REF", `<fragment> using must be alias:fragment`);
}
- const inputs: { readonly name: string; readonly from: string }[] = [];
+ const inputs: RunFragmentInstance["inputs"][number][] = [];
const exports: string[] = [];
for (const child of elements(element)) {
if (child.name === "input") {
- exactAttributes(child, ["name", "from"]);
+ exactAttributes(child, ["name", "from", "value"]);
empty(child);
- inputs.push({ name: stringAttribute(child, "name")!, from: stringAttribute(child, "from")! });
+ const name = stringAttribute(child, "name")!;
+ const from = stringAttribute(child, "from", false);
+ const value = stringAttribute(child, "value", false);
+ if ((from === undefined) === (value === undefined)) {
+ fail(child, "RUN_FRAGMENT_INPUT", `<input> ${name} requires exactly one of from or value`);
+ }
+ inputs.push(from === undefined ? { name, value: scalar(value!) } : { name, from });
continue;
}
if (child.name === "export") {
diff --git a/packages/run-markup/test/run.test.ts b/packages/run-markup/test/run.test.ts
index 13ca9274..2ce3c52a 100644
--- a/packages/run-markup/test/run.test.ts
+++ b/packages/run-markup/test/run.test.ts
@@ -161,6 +161,54 @@ test("one multi-export Fragment declaration remains one execution", async () =>
assert.equal(state.plan.steps.filter((item) => item.producer.name === "preview").length, 1);
});
+test("a Run Fragment may own an ordinary literal input", async () => {
+ const compiled = await compileDocument(`<svrun version="1">
+ <author source="./main.svml"/>
+ <import from="@example/run-preview" as="preview"/>
+ <target output="left"/>
+ <fragment id="card" using="preview:shared"><input name="prompt" value="five seconds"/></fragment>
+ <fragment id="unused" using="preview:shared"><input name="prompt" value="not selected"/></fragment>
+ <satisfy output="left" candidate="card.first"/>
+ </svrun>`);
+ const compilation = fixture();
+ const fragments = new RunFragmentRegistry();
+ fragments.register({ name: "@example/run-preview", fragments: { shared: previewFragment() } });
+ const run = await resolveRunDocument(compiled.document, { compilation, fragments });
+ assert.deepEqual(run.graph.records, [
+ {
+ id: "record:run:card:prompt",
+ type: promptType,
+ value: { kind: "inline", value: "five seconds" },
+ },
+ {
+ id: "record:run:unused:prompt",
+ type: promptType,
+ value: { kind: "inline", value: "not selected" },
+ },
+ ]);
+ const state = realize(compilation, run);
+ assert.equal(state.records.find((item) => item.id === "record:run:card:prompt")?.value.kind, "inline");
+ assert.equal(state.records.some((item) => item.id === "record:run:unused:prompt"), false);
+});
+
+test("Run literal syntax decodes numbers and requires one source of value", () => {
+ const parsed = parseRunDocument("duration.svrun", `<svrun version="1">
+ <author source="./main.svml"/>
+ <import from="@example/run-preview" as="preview"/>
+ <target output="left"/>
+ <fragment id="card" using="preview:shared"><input name="prompt" value="5"/></fragment>
+ <satisfy output="left" candidate="card.first"/>
+ </svrun>`);
+ const input = parsed.candidates.find((item) => item.kind === "fragment")?.inputs[0];
+ assert.deepEqual(input, { name: "prompt", value: 5 });
+ assert.throws(() => parseRunDocument("bad.svrun", `<svrun version="1">
+ <author source="./main.svml"/>
+ <import from="@example/run-preview" as="preview"/>
+ <target output="left"/>
+ <fragment id="card" using="preview:shared"><input name="prompt" from="prompt" value="5"/></fragment>
+ </svrun>`), /requires exactly one of from or value/u);
+});
+
test("a Provided Value is an ordinary zero-input Candidate", async () => {
const compiled = await compileDocument(`<svrun version="1">
<author source="./main.svml"/>
diff --git a/packages/run/src/graph.ts b/packages/run/src/graph.ts
index 0936358b..817e0176 100644
--- a/packages/run/src/graph.ts
+++ b/packages/run/src/graph.ts
@@ -17,6 +17,7 @@ function assert(condition: unknown, code: string, message: string): asserts cond
function content(graph: RunGraph): RunGraph {
return {
format: "hypit.run-graph@1",
+ records: [...graph.records].sort((left, right) => left.id.localeCompare(right.id)),
candidates: [...graph.candidates].sort((left, right) => left.id.localeCompare(right.id)),
operations: [...graph.operations].sort((left, right) => left.id.localeCompare(right.id)),
satisfactions: [...graph.satisfactions].sort((left, right) => left.output.localeCompare(right.output)),
@@ -32,6 +33,13 @@ export function sealRunGraph(input: Omit<RunGraph, "format">): RunGraph {
export function verifyRunGraph(graph: RunGraph): void {
assert(graph.format === "hypit.run-graph@1", "UNSUPPORTED_RUN_GRAPH", "unsupported Run Graph");
+ const recordIds = new Set<string>();
+ for (const record of graph.records) {
+ assert(record.id.length > 0, "EMPTY_RUN_RECORD", "Run Graph Record id is empty");
+ assert(!recordIds.has(record.id), "DUPLICATE_RUN_RECORD", `Run Graph repeats Record ${record.id}`);
+ assert(record.value.kind === "inline", "INVALID_RUN_RECORD", `Run Graph Record ${record.id} must be inline`);
+ recordIds.add(record.id);
+ }
const candidateIds = new Set<string>();
for (const candidate of graph.candidates) {
assert(!candidateIds.has(candidate.id), "DUPLICATE_RUN_CANDIDATE", `Run Graph repeats Candidate ${candidate.id}`);
diff --git a/packages/run/src/resolve.ts b/packages/run/src/resolve.ts
index dd02444d..d94e0150 100644
--- a/packages/run/src/resolve.ts
+++ b/packages/run/src/resolve.ts
@@ -3,12 +3,13 @@ import {
exportRunFragment,
resolveCompiledSourceExport,
} from "@hypit/elaborator";
-import { sameType } from "@hypit/protocol";
+import { canonicalize, sameType } from "@hypit/protocol";
import type {
Candidate,
GraphValueRef,
ModuleRef,
OperationNode,
+ TypedRecord,
} from "@hypit/protocol";
import {
createProvidedCandidate,
@@ -75,6 +76,7 @@ export async function resolveRunDocument(
const imports = new Map(document.imports.map((item) => [item.as, item.from]));
const candidates = new Map<string, Candidate>();
const operations = new Map<string, OperationNode>();
+ const records = new Map<string, TypedRecord>();
const addCandidate = (candidate: Candidate): void => {
if (candidates.has(candidate.id)) throw new Error(`Run Candidate ${candidate.id} is declared twice`);
candidates.set(candidate.id, candidate);
@@ -150,10 +152,26 @@ export async function resolveRunDocument(
if (fragment === undefined) {
throw new Error(`${packageName} exports no Run Fragment ${declaration.using.name}`);
}
+ const inputDeclarations = new Map(fragment.inputs.map((item) => [item.name, item]));
+ const fragmentInputs = Object.fromEntries(declaration.inputs.map((item): [string, GraphValueRef] => {
+ if ("from" in item) return [item.name, authorRef(context, item.from)];
+ const inputDeclaration = inputDeclarations.get(item.name);
+ if (inputDeclaration === undefined) {
+ throw new Error(`Run Fragment ${declaration.id} binds undeclared input ${item.name}`);
+ }
+ const id = `record:run:${declaration.id}:${item.name}`;
+ if (records.has(id)) throw new Error(`Run Fragment literal ${declaration.id}.${item.name} is declared twice`);
+ records.set(id, {
+ id,
+ type: inputDeclaration.type,
+ value: { kind: "inline", value: canonicalize(item.value) },
+ });
+ return [item.name, { kind: "record", id }];
+ }));
const instance = elaborateGraphFragment(context.compilation.program, fragment, {
id: `run:${declaration.id}`,
fragment: fragment.id,
- inputs: Object.fromEntries(declaration.inputs.map((item) => [item.name, authorRef(context, item.from)])),
+ inputs: fragmentInputs,
});
const contribution = exportRunFragment(instance, declaration.exports);
for (const candidate of contribution.candidates) addCandidate(candidate);
@@ -179,6 +197,7 @@ export async function resolveRunDocument(
const resolvedCandidates = [...candidates.values()];
const resolvedOperations = [...operations.values()];
const graph = sealRunGraph({
+ records: [...records.values()],
candidates: resolvedCandidates,
operations: resolvedOperations,
satisfactions,
diff --git a/packages/run/src/types.ts b/packages/run/src/types.ts
index 25a6b904..3a1c5d7b 100644
--- a/packages/run/src/types.ts
+++ b/packages/run/src/types.ts
@@ -4,6 +4,7 @@ import type {
} from "@hypit/elaborator";
import type {
Candidate,
+ CanonicalValue,
OperationNode,
RunGraph,
Satisfaction,
@@ -56,11 +57,17 @@ export type RunBuildRecord = {
readonly output: string;
};
-export type RunFragmentInput = {
- readonly name: string;
- /** Public author-source export. */
- readonly from: string;
-};
+export type RunFragmentInput =
+ | {
+ readonly name: string;
+ /** Public author-source export. */
+ readonly from: string;
+ }
+ | {
+ readonly name: string;
+ /** An ordinary scalar owned by this Run, typed by the Fragment input declaration. */
+ readonly value: CanonicalValue;
+ };
export type RunFragmentInstance = {
readonly kind: "fragment";
diff --git a/packages/seedance/src/activation.ts b/packages/seedance/src/activation.ts
index 912cb520..c4bda476 100644
--- a/packages/seedance/src/activation.ts
+++ b/packages/seedance/src/activation.ts
@@ -17,7 +17,6 @@ export const hypitPackage = {
components: [seedanceComponent],
hostFacets: [
seedanceDefinition.hostFacet,
- ...(seedanceDefinition.runFragmentFacet === undefined ? [] : [seedanceDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({
module: seedanceModuleRef,
declaration: seedanceMarkupSurfaces.find((item) => item.name === "text-video")!,
diff --git a/packages/seedream/src/activation.ts b/packages/seedream/src/activation.ts
index 1bccf402..2ccbedad 100644
--- a/packages/seedream/src/activation.ts
+++ b/packages/seedream/src/activation.ts
@@ -4,7 +4,6 @@ import { seedreamComponent, seedreamDefinition, seedreamManifest, seedreamModule
import { decodeSeedreamReferenceImageSurface, decodeSeedreamTextImageSurface } from "./surface.js";
export const hypitPackage = { format: "hypit.node-package@1" as const, modules: [{ manifest: seedreamManifest }], components: [seedreamComponent], hostFacets: [
seedreamDefinition.hostFacet,
- ...(seedreamDefinition.runFragmentFacet === undefined ? [] : [seedreamDefinition.runFragmentFacet]),
createMarkupSurfaceHostFacet({ module: seedreamModuleRef,
declaration: seedreamMarkupSurfaces.find((item) => item.name === "text-image")!, handler: decodeSeedreamTextImageSurface }),
createMarkupSurfaceHostFacet({ module: seedreamModuleRef,
diff --git a/packages/stand-in/README.md b/packages/stand-in/README.md
index d06f5082..a96b47ec 100644
--- a/packages/stand-in/README.md
+++ b/packages/stand-in/README.md
@@ -1,29 +1,23 @@
# `@hypit/stand-in`
-A stand-in is what a generated output looks like before it is generated: a designed card, drawn
-instead of bought, that the author selects as a Candidate for that output in an ordinary Run. The
-card is derived from the model's own request — kind, frame, model, duration, prompt — so it follows
-the Source when the Source changes, and it goes through the same inspect and normalize steps as the
-real output would.
+A stand-in is an ordinary generic Candidate selected by a Run in place of an output that is not
+needed yet. It knows nothing about the displaced Candidate's Producer, model, prompt or request.
+The Run supplies only the media shape the downstream composition needs, and the resulting file goes
+through the same inspect and normalize steps as any other file.
-This module owns the vocabulary only: the `StandInCardRequest` type and the `draw-card` capability.
-Every exact-model package built with `@hypit/model-kit` exports one Run Fragment per model that
-turns its draft into that request; a local media Provider draws the card (`@hypit/media-execution`).
-Nothing here is a mock: the Provider that serves `draw-card` really draws a card, prices it as
-local work, and never pretends to be the model.
+This package exports three Run Fragments: `image`, `video` and `silence`. A local media Provider
+materializes their deterministic files. Model packages do not know these Fragments exist.
```xml
-<import from="@hypit/seedance@1" as="seedance"/>
-<fragment id="kitchen-card" using="seedance:mini-stand-in">
- <input name="draft" from="kitchen.draft"/>
- <input name="prompt" from="kitchen-direction"/>
+<import from="@hypit/stand-in@1" as="stand-in"/>
+<fragment id="kitchen-card" using="stand-in:video">
+ <input name="canvas" from="vertical"/>
+ <input name="duration" value="5"/>
<input name="clock" from="clock"/>
</fragment>
<satisfy output="kitchen.video" candidate="kitchen-card.video"/>
```
-Remove the `satisfy` line and build again to buy the real output. The card carries a diagonal
-STAND-IN watermark, its kind in large type, the model, the frame and duration, an excerpt of the
-prompt, and — for video — a running timecode and progress bar, so a cut that lands on the wrong
-frame is visible in the preview. Text on the card is drawn with a built-in bitmap font: Latin
-letters, digits and common punctuation; other characters appear as boxes.
+Remove the `satisfy` line to select the Source's primary Candidate again. The card carries a
+diagonal STAND-IN watermark, its kind, frame and duration and — for video — a running timecode and
+progress bar, so a cut that lands on the wrong frame remains visible.
diff --git a/packages/stand-in/package.json b/packages/stand-in/package.json
index 2966fa97..4b95a955 100644
--- a/packages/stand-in/package.json
+++ b/packages/stand-in/package.json
@@ -13,7 +13,12 @@
"dependencies": {
"@hypit/artifact": "workspace:*",
"@hypit/component-kit": "workspace:*",
+ "@hypit/elaborator": "workspace:*",
"@hypit/media": "workspace:*",
- "@hypit/protocol": "workspace:*"
+ "@hypit/program-space": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/run": "workspace:*",
+ "@hypit/spatial": "workspace:*",
+ "@hypit/speech": "workspace:*"
}
}
diff --git a/packages/stand-in/src/activation.ts b/packages/stand-in/src/activation.ts
index d38455b1..ee16c204 100644
--- a/packages/stand-in/src/activation.ts
+++ b/packages/stand-in/src/activation.ts
@@ -1,10 +1,19 @@
import { standInComponent, standInManifest } from "./index.js";
+import { createRunFragmentHostFacet } from "@hypit/run";
+import { standInImageFragment, standInSilenceFragment, standInVideoFragment } from "./fragment.js";
export const hypitPackage = {
format: "hypit.node-package@1" as const,
modules: [{ manifest: standInManifest }],
components: [standInComponent],
- hostFacets: [],
+ hostFacets: [createRunFragmentHostFacet({
+ name: "@hypit/stand-in@1",
+ fragments: {
+ image: standInImageFragment,
+ video: standInVideoFragment,
+ silence: standInSilenceFragment,
+ },
+ })],
};
export default hypitPackage;
diff --git a/packages/stand-in/src/fragment.ts b/packages/stand-in/src/fragment.ts
new file mode 100644
index 00000000..2d00d022
--- /dev/null
+++ b/packages/stand-in/src/fragment.ts
@@ -0,0 +1,47 @@
+import { artifactTypes } from "@hypit/artifact";
+import { sealGraphFragment } from "@hypit/elaborator";
+import { programSpaceTypes } from "@hypit/program-space";
+import { spatialTypes } from "@hypit/spatial";
+import { speechTypes } from "@hypit/speech";
+
+import { standInProducers } from "./manifest.js";
+
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (name: string) => ({ kind: "fragment-operation" as const, operation: name });
+
+export const standInImageFragment = sealGraphFragment({
+ inputs: [{ name: "canvas", type: spatialTypes.canvas }],
+ operations: [{
+ id: "image",
+ producer: standInProducers.image,
+ inputs: { canvas: input("canvas") },
+ result: { kind: "need", name: "image" },
+ }],
+ exports: [{ name: "image", type: artifactTypes.blob, root: operation("image") }],
+});
+
+export const standInVideoFragment = sealGraphFragment({
+ inputs: [
+ { name: "canvas", type: spatialTypes.canvas },
+ { name: "duration", type: speechTypes.duration },
+ { name: "clock", type: programSpaceTypes.clock },
+ ],
+ operations: [{
+ id: "video",
+ producer: standInProducers.video,
+ inputs: { canvas: input("canvas"), duration: input("duration"), clock: input("clock") },
+ result: { kind: "need", name: "video" },
+ }],
+ exports: [{ name: "video", type: artifactTypes.blob, root: operation("video") }],
+});
+
+export const standInSilenceFragment = sealGraphFragment({
+ inputs: [{ name: "duration", type: speechTypes.duration }],
+ operations: [{
+ id: "silence",
+ producer: standInProducers.silence,
+ inputs: { duration: input("duration") },
+ result: { kind: "need", name: "audio" },
+ }],
+ exports: [{ name: "audio", type: artifactTypes.blob, root: operation("silence") }],
+});
diff --git a/packages/stand-in/src/index.ts b/packages/stand-in/src/index.ts
index 0be29e75..4b0bfecc 100644
--- a/packages/stand-in/src/index.ts
+++ b/packages/stand-in/src/index.ts
@@ -1,32 +1,35 @@
import type { ComponentPackage } from "@hypit/component-kit";
import type { MediaRational } from "@hypit/media";
+import { assertProgramClockIdentity } from "@hypit/program-space";
+import type { ProgramClock } from "@hypit/program-space";
import { canonicalize } from "@hypit/protocol";
import type { CanonicalValue, StoredValue } from "@hypit/protocol";
+import { assertCanvasSpace } from "@hypit/spatial";
+import type { CanvasSpace } from "@hypit/spatial";
+import { assertSpeechDurationIdentity } from "@hypit/speech";
+import type { SpeechDuration } from "@hypit/speech";
-import { standInTypes } from "./manifest.js";
+import { standInProducers, standInTypes } from "./manifest.js";
export {
standInCapabilities,
standInDependency,
standInManifest,
standInModuleRef,
+ standInProducers,
standInTypes,
} from "./manifest.js";
+export { standInImageFragment, standInSilenceFragment, standInVideoFragment } from "./fragment.js";
/**
- * What a stand-in card says about the output it stands in for. Every field is derived from the
- * declaration the card replaces — the model's own request — never restated by hand, so the card
- * follows the Source when the Source changes.
+ * The minimum visible shape of a generic stand-in. It says nothing about the
+ * Producer or model whose output a Run may choose to replace with it.
*/
export type StandInCardRequest = {
readonly kind: "image" | "video";
/** Even, positive pixel dimensions of the card, the frame the real output would have. */
readonly width: number;
readonly height: number;
- /** The exact model that would have generated the output. */
- readonly model: string;
- /** The prompt the model would have received; the card prints an excerpt. */
- readonly prompt: string;
/** A video card runs for exactly this many frames at this clock and carries a running timecode. */
readonly video?: {
readonly frameRate: MediaRational;
@@ -36,6 +39,12 @@ export type StandInCardRequest = {
};
};
+export type StandInSilenceRequest = {
+ readonly sampleRate: 48_000;
+ readonly channels: 2;
+ readonly sampleFrames: number;
+};
+
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
@@ -52,8 +61,6 @@ export function verifyStandInCardRequest(value: unknown): asserts value is Stand
assert(item.kind === "image" || item.kind === "video", "StandInCardRequest kind must be image or video");
evenPositive(item.width, "StandInCardRequest width");
evenPositive(item.height, "StandInCardRequest height");
- assert(typeof item.model === "string" && item.model.trim().length > 0, "StandInCardRequest model must name the model");
- assert(typeof item.prompt === "string", "StandInCardRequest prompt must be a string");
if (item.kind === "video") {
const video = item.video as Record<string, unknown> | undefined;
assert(video !== undefined && typeof video === "object", "a video StandInCardRequest needs its frame domain");
@@ -69,6 +76,14 @@ export function verifyStandInCardRequest(value: unknown): asserts value is Stand
}
}
+export function verifyStandInSilenceRequest(value: unknown): asserts value is StandInSilenceRequest {
+ assert(value !== null && typeof value === "object" && !Array.isArray(value), "StandInSilenceRequest must be an object");
+ const item = value as Record<string, unknown>;
+ assert(item.sampleRate === 48_000 && item.channels === 2, "StandInSilenceRequest must be 48 kHz stereo");
+ assert(Number.isSafeInteger(item.sampleFrames) && (item.sampleFrames as number) > 0,
+ "StandInSilenceRequest sampleFrames must be a positive integer");
+}
+
export function sealStandInCardRequest(value: StandInCardRequest): StandInCardRequest {
verifyStandInCardRequest(value);
return canonicalize(value as unknown as CanonicalValue) as unknown as StandInCardRequest;
@@ -79,9 +94,67 @@ function inline(value: StoredValue, subject: string): unknown {
return value.value;
}
+function typedInline<T>(value: StoredValue, subject: string): T {
+ return inline(value, subject) as T;
+}
+
+function exactFrameCount(duration: SpeechDuration, clock: ProgramClock): number {
+ const frames = duration * clock.frameRate.numerator / clock.frameRate.denominator;
+ const rounded = Math.round(frames);
+ assert(Number.isSafeInteger(rounded) && rounded > 0 && Math.abs(frames - rounded) < 1e-7,
+ "Stand-in video duration must end on a whole ProgramClock frame");
+ return rounded;
+}
+
export const standInComponent = {
- validators: [{
- type: standInTypes.cardRequest,
- handler: ({ value }) => { verifyStandInCardRequest(inline(value, "StandInCardRequest")); },
- }],
+ validators: [
+ {
+ type: standInTypes.cardRequest,
+ handler: ({ value }) => { verifyStandInCardRequest(inline(value, "StandInCardRequest")); },
+ },
+ {
+ type: standInTypes.silenceRequest,
+ handler: ({ value }) => { verifyStandInSilenceRequest(inline(value, "StandInSilenceRequest")); },
+ },
+ ],
+ producers: [
+ {
+ producer: standInProducers.image,
+ handler: ({ inputs }) => {
+ const canvas = typedInline<CanvasSpace>(inputs.canvas!.value, "stand-in image Canvas");
+ assertCanvasSpace(canvas);
+ return { outputs: {}, needs: { image: canonicalize(sealStandInCardRequest({
+ kind: "image", width: canvas.widthPx, height: canvas.heightPx,
+ })) } };
+ },
+ },
+ {
+ producer: standInProducers.video,
+ handler: ({ inputs }) => {
+ const canvas = typedInline<CanvasSpace>(inputs.canvas!.value, "stand-in video Canvas");
+ const duration = typedInline<SpeechDuration>(inputs.duration!.value, "stand-in video duration");
+ const clock = typedInline<ProgramClock>(inputs.clock!.value, "stand-in video clock");
+ assertCanvasSpace(canvas);
+ assertSpeechDurationIdentity(duration);
+ assertProgramClockIdentity(clock);
+ return { outputs: {}, needs: { video: canonicalize(sealStandInCardRequest({
+ kind: "video",
+ width: canvas.widthPx,
+ height: canvas.heightPx,
+ video: { frameRate: clock.frameRate, frameCount: exactFrameCount(duration, clock), audio: "silence" },
+ })) } };
+ },
+ },
+ {
+ producer: standInProducers.silence,
+ handler: ({ inputs }) => {
+ const duration = typedInline<SpeechDuration>(inputs.duration!.value, "stand-in silence duration");
+ assertSpeechDurationIdentity(duration);
+ const sampleFrames = Math.round(duration * 48_000);
+ const request: StandInSilenceRequest = { sampleRate: 48_000, channels: 2, sampleFrames };
+ verifyStandInSilenceRequest(request);
+ return { outputs: {}, needs: { audio: canonicalize(request) } };
+ },
+ },
+ ],
} satisfies ComponentPackage;
diff --git a/packages/stand-in/src/manifest.ts b/packages/stand-in/src/manifest.ts
index 0a1f5b18..34a0591c 100644
--- a/packages/stand-in/src/manifest.ts
+++ b/packages/stand-in/src/manifest.ts
@@ -1,24 +1,58 @@
import { artifactDependency, artifactTypes } from "@hypit/artifact";
+import { programSpaceDependency, programSpaceTypes } from "@hypit/program-space";
import type { CapabilityRef, ModuleManifest, TypeRef } from "@hypit/protocol";
+import { spatialDependency, spatialTypes } from "@hypit/spatial";
+import { speechDependency, speechTypes } from "@hypit/speech";
export const standInModuleRef = { name: "@hypit/stand-in", version: "1" } as const;
export const standInDependency = { module: standInModuleRef } as const;
export const standInTypes = {
cardRequest: { module: standInModuleRef, name: "StandInCardRequest" },
+ silenceRequest: { module: standInModuleRef, name: "StandInSilenceRequest" },
} satisfies Record<string, TypeRef>;
/** Draw one stand-in card. Served by a local media Provider; it draws, it never generates. */
export const standInCapabilities = {
drawCard: { module: standInModuleRef, name: "draw-card" },
+ drawSilence: { module: standInModuleRef, name: "draw-silence" },
} satisfies Record<string, CapabilityRef>;
+export const standInProducers = {
+ image: { module: standInModuleRef, name: "stand-in-image" },
+ video: { module: standInModuleRef, name: "stand-in-video" },
+ silence: { module: standInModuleRef, name: "stand-in-silence" },
+} as const;
+
export const standInManifest: ModuleManifest = {
format: "hypit.module@1",
name: standInModuleRef.name,
version: standInModuleRef.version,
- dependencies: [artifactDependency],
+ dependencies: [artifactDependency, programSpaceDependency, spatialDependency, speechDependency],
types: Object.values(standInTypes).map((item) => ({ name: item.name })),
capabilities: Object.values(standInCapabilities).map((item) => ({ name: item.name, returns: artifactTypes.blob })),
- producers: [],
+ producers: [
+ {
+ name: standInProducers.image.name,
+ inputs: [{ name: "canvas", type: spatialTypes.canvas }],
+ outputs: [],
+ needs: [{ name: "image", capability: standInCapabilities.drawCard, returns: artifactTypes.blob }],
+ },
+ {
+ name: standInProducers.video.name,
+ inputs: [
+ { name: "canvas", type: spatialTypes.canvas },
+ { name: "duration", type: speechTypes.duration },
+ { name: "clock", type: programSpaceTypes.clock },
+ ],
+ outputs: [],
+ needs: [{ name: "video", capability: standInCapabilities.drawCard, returns: artifactTypes.blob }],
+ },
+ {
+ name: standInProducers.silence.name,
+ inputs: [{ name: "duration", type: speechTypes.duration }],
+ outputs: [],
+ needs: [{ name: "audio", capability: standInCapabilities.drawSilence, returns: artifactTypes.blob }],
+ },
+ ],
};
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 82d13c72..ba5aae55 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1459,9 +1459,6 @@ importers:
'@hypit/speech':
specifier: workspace:*
version: link:../speech
- '@hypit/stand-in':
- specifier: workspace:*
- version: link:../stand-in
packages/grok-imagine:
dependencies:
@@ -1837,18 +1834,9 @@ importers:
'@hypit/host':
specifier: workspace:*
version: link:../host
- '@hypit/program-space':
- specifier: workspace:*
- version: link:../program-space
'@hypit/protocol':
specifier: workspace:*
version: link:../protocol
- '@hypit/run':
- specifier: workspace:*
- version: link:../run
- '@hypit/stand-in':
- specifier: workspace:*
- version: link:../stand-in
'@hypit/text':
specifier: workspace:*
version: link:../text
@@ -2944,12 +2932,27 @@ importers:
'@hypit/component-kit':
specifier: workspace:*
version: link:../component-kit
+ '@hypit/elaborator':
+ specifier: workspace:*
+ version: link:../elaborator
'@hypit/media':
specifier: workspace:*
version: link:../media
+ '@hypit/program-space':
+ specifier: workspace:*
+ version: link:../program-space
'@hypit/protocol':
specifier: workspace:*
version: link:../protocol
+ '@hypit/run':
+ specifier: workspace:*
+ version: link:../run
+ '@hypit/spatial':
+ specifier: workspace:*
+ version: link:../spatial
+ '@hypit/speech':
+ specifier: workspace:*
+ version: link:../speech
packages/store-sqlite:
dependencies: