Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat: keep generated media on graph edges

kashorinbbf536968f84
diff --git a/examples/talking-film-graph-check/svml.packages.lock b/examples/talking-film-graph-check/svml.packages.lock
index 742fee0b..e4b565ad 100644
--- a/examples/talking-film-graph-check/svml.packages.lock
+++ b/examples/talking-film-graph-check/svml.packages.lock
@@ -604,7 +604,7 @@
{
"name": "@narratage/generation",
"version": "0.0.0-dev",
- "digest": "sha256:4ceaf545fa645de80873c5cc3b9df7655474a262057f805e85eca2679025536e"
+ "digest": "sha256:aa8125353ac0e60222bd038503fd3787bd7c942bccd794c013bf5220eb454bc9"
},
{
"name": "@narratage/host",
@@ -634,7 +634,7 @@
{
"name": "@narratage/model-kit",
"version": "0.0.0-dev",
- "digest": "sha256:debadc91af0b1c706f6d44214be281fbf05a53b0d53d027066d06a1a8752e477"
+ "digest": "sha256:b5ce0f8b10756e801827f119b3335b22b8aab1ed1765f6a6261457355506bcab"
},
{
"name": "@narratage/narrative",
@@ -674,7 +674,7 @@
{
"name": "@narratage/seedance",
"version": "0.0.0-dev",
- "digest": "sha256:26c103a2093171da0f30e1676b76f3667518114df57110fe0885e16241c05952"
+ "digest": "sha256:e218a3c54118c75a21c080cc20a4dd410c78223000100a009dfe270c1429b03d"
},
{
"name": "@narratage/semantic-map",
@@ -832,7 +832,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:67fa8f4123972a33bd2d0e1cc6126a9ed8d3d09cc2745aa058888b735fc85570"
+ "closureDigest": "sha256:ae96c20681e676094f7a411ce82a4b2d9f835a4fffa189dd7b6550c321166d94"
},
{
"specifier": "@narratage/hyperframes",
@@ -921,8 +921,8 @@
"name": "@narratage/seedance",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:3b9cebaa0503cb716c3d8314a6cf837b13dd5d88c2b09fd9604ab38e67d7af5e",
- "closureDigest": "sha256:d8f3669cc7a75d22e30a4e3b304f6a87ff9d33920199e50c043d6ed0e6d4e2be"
+ "facetsDigest": "sha256:c1f9cb32b1343f2a5b095e43c5a9501b6d015fe064b47af480fa6d3d10e1782e",
+ "closureDigest": "sha256:3d75f4413dd8b4435ce8e3e96d1a6881ff11872aa0b2b7dd3e7b4b8e6469903a"
},
{
"specifier": "@narratage/semantic-map",
@@ -1033,5 +1033,5 @@
"closureDigest": "sha256:1dd3e16f58411fd77f2daf4a945781a784618cffe7512d83d6181f701c39cadb"
}
],
- "digest": "sha256:53b9f0366ff63d096841760fe4c6b7c5e71a50005564f57eff44539130f45faa"
+ "digest": "sha256:bb5c2de4ed05234f9bdbc3ee6b966479d10c0dfee77e49d27a66571fddfc7de0"
}
diff --git a/examples/talking-film-live/svml.packages.lock b/examples/talking-film-live/svml.packages.lock
index 1f846e86..8ccbb4d5 100644
--- a/examples/talking-film-live/svml.packages.lock
+++ b/examples/talking-film-live/svml.packages.lock
@@ -604,7 +604,7 @@
{
"name": "@narratage/generation",
"version": "0.0.0-dev",
- "digest": "sha256:4ceaf545fa645de80873c5cc3b9df7655474a262057f805e85eca2679025536e"
+ "digest": "sha256:aa8125353ac0e60222bd038503fd3787bd7c942bccd794c013bf5220eb454bc9"
},
{
"name": "@narratage/host",
@@ -634,7 +634,7 @@
{
"name": "@narratage/model-kit",
"version": "0.0.0-dev",
- "digest": "sha256:debadc91af0b1c706f6d44214be281fbf05a53b0d53d027066d06a1a8752e477"
+ "digest": "sha256:b5ce0f8b10756e801827f119b3335b22b8aab1ed1765f6a6261457355506bcab"
},
{
"name": "@narratage/narrative",
@@ -674,7 +674,7 @@
{
"name": "@narratage/seedance",
"version": "0.0.0-dev",
- "digest": "sha256:26c103a2093171da0f30e1676b76f3667518114df57110fe0885e16241c05952"
+ "digest": "sha256:e218a3c54118c75a21c080cc20a4dd410c78223000100a009dfe270c1429b03d"
},
{
"name": "@narratage/semantic-map",
@@ -832,7 +832,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:67fa8f4123972a33bd2d0e1cc6126a9ed8d3d09cc2745aa058888b735fc85570"
+ "closureDigest": "sha256:ae96c20681e676094f7a411ce82a4b2d9f835a4fffa189dd7b6550c321166d94"
},
{
"specifier": "@narratage/hyperframes",
@@ -912,8 +912,8 @@
"name": "@narratage/seedance",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:3b9cebaa0503cb716c3d8314a6cf837b13dd5d88c2b09fd9604ab38e67d7af5e",
- "closureDigest": "sha256:d8f3669cc7a75d22e30a4e3b304f6a87ff9d33920199e50c043d6ed0e6d4e2be"
+ "facetsDigest": "sha256:c1f9cb32b1343f2a5b095e43c5a9501b6d015fe064b47af480fa6d3d10e1782e",
+ "closureDigest": "sha256:3d75f4413dd8b4435ce8e3e96d1a6881ff11872aa0b2b7dd3e7b4b8e6469903a"
},
{
"specifier": "@narratage/semantic-map",
@@ -1015,5 +1015,5 @@
"closureDigest": "sha256:1dd3e16f58411fd77f2daf4a945781a784618cffe7512d83d6181f701c39cadb"
}
],
- "digest": "sha256:6c2f003b40c9824af27dfbbf08015ee9a10f1d79231167faeb07d0c7c4389c8f"
+ "digest": "sha256:b6cc9ac228600f6e32a7487547765f67083213971fabdc3f2290263581995702"
}
diff --git a/examples/talking-head-aroll/svml.packages.lock b/examples/talking-head-aroll/svml.packages.lock
index 010674c4..b45e4895 100644
--- a/examples/talking-head-aroll/svml.packages.lock
+++ b/examples/talking-head-aroll/svml.packages.lock
@@ -604,7 +604,7 @@
{
"name": "@narratage/generation",
"version": "0.0.0-dev",
- "digest": "sha256:4ceaf545fa645de80873c5cc3b9df7655474a262057f805e85eca2679025536e"
+ "digest": "sha256:aa8125353ac0e60222bd038503fd3787bd7c942bccd794c013bf5220eb454bc9"
},
{
"name": "@narratage/host",
@@ -634,7 +634,7 @@
{
"name": "@narratage/model-kit",
"version": "0.0.0-dev",
- "digest": "sha256:debadc91af0b1c706f6d44214be281fbf05a53b0d53d027066d06a1a8752e477"
+ "digest": "sha256:b5ce0f8b10756e801827f119b3335b22b8aab1ed1765f6a6261457355506bcab"
},
{
"name": "@narratage/narrative",
@@ -679,12 +679,12 @@
{
"name": "@narratage/seedance-speaker",
"version": "0.0.0-dev",
- "digest": "sha256:0fe589c485b9ba00257594d3161e6606703a08d84090798eec3906cd46b1f520"
+ "digest": "sha256:4141de3dcd31bf05be36204bdc2112aa2f5b849fd4a062df5e513ef22460e963"
},
{
"name": "@narratage/seedance",
"version": "0.0.0-dev",
- "digest": "sha256:26c103a2093171da0f30e1676b76f3667518114df57110fe0885e16241c05952"
+ "digest": "sha256:e218a3c54118c75a21c080cc20a4dd410c78223000100a009dfe270c1429b03d"
},
{
"name": "@narratage/semantic-map",
@@ -837,7 +837,7 @@
"version": "0.0.0-dev"
},
"facetsDigest": "sha256:d74a57a3210267e665366513324db5c313b3ca0f770da66e74cc38aee5048efb",
- "closureDigest": "sha256:67fa8f4123972a33bd2d0e1cc6126a9ed8d3d09cc2745aa058888b735fc85570"
+ "closureDigest": "sha256:ae96c20681e676094f7a411ce82a4b2d9f835a4fffa189dd7b6550c321166d94"
},
{
"specifier": "@narratage/hyperframes",
@@ -926,8 +926,8 @@
"name": "@narratage/seedance",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:3b9cebaa0503cb716c3d8314a6cf837b13dd5d88c2b09fd9604ab38e67d7af5e",
- "closureDigest": "sha256:d8f3669cc7a75d22e30a4e3b304f6a87ff9d33920199e50c043d6ed0e6d4e2be"
+ "facetsDigest": "sha256:c1f9cb32b1343f2a5b095e43c5a9501b6d015fe064b47af480fa6d3d10e1782e",
+ "closureDigest": "sha256:3d75f4413dd8b4435ce8e3e96d1a6881ff11872aa0b2b7dd3e7b4b8e6469903a"
},
{
"specifier": "@narratage/seedance-speaker",
@@ -935,8 +935,8 @@
"name": "@narratage/seedance-speaker",
"version": "0.0.0-dev"
},
- "facetsDigest": "sha256:82648b54b0e363bae552ad852bc4fb261088e20b90e7cf2ab94471f0be68ba35",
- "closureDigest": "sha256:f09074e9d2145c724f174958dc86e0398a600e2109750da1306339a048df6899"
+ "facetsDigest": "sha256:66aa6e0a9ae24f7c6a04bd45e79266a8195292d5b6e3c4069fd8f465f3c5386a",
+ "closureDigest": "sha256:2157e4331d10cc2b12e16efdac41a7951827fde88242bef2e99e032294596fa7"
},
{
"specifier": "@narratage/semantic-map",
@@ -1029,5 +1029,5 @@
"closureDigest": "sha256:1dd3e16f58411fd77f2daf4a945781a784618cffe7512d83d6181f701c39cadb"
}
],
- "digest": "sha256:ec4435e504b092818c851290ff4e4c1528f54e98ed88e9315b180844f4a488dd"
+ "digest": "sha256:34569197250bc2b943f156fa3d7e41267b4defd2066e147eb38e702c553b3f36"
}
diff --git a/packages/generation/src/index.ts b/packages/generation/src/index.ts
index 4c31d71a..ddd884f2 100644
--- a/packages/generation/src/index.ts
+++ b/packages/generation/src/index.ts
@@ -54,14 +54,29 @@ export type {
GenerationScalarPort,
} from "./ports.js";
export {
+ bindGenerationMedia,
+ finalizeGenerationRequestDraft,
+ GENERATION_MEDIA_BINDING_V1,
+ GENERATION_REQUEST_DRAFT_V1,
GENERATION_REQUEST_V1,
+ mediaBindingSchemaFromPort,
portsObjectSchema,
+ requestDraftSchemaFromPorts,
requestSchemaFromPorts,
+ sealGenerationMediaBinding,
sealGenerationPortRequest,
+ sealGenerationRequestDraft,
+ verifyGenerationMediaBinding,
verifyPortsAgainstTable,
+ verifyRequestDraftAgainstPorts,
verifyRequestAgainstPorts,
} from "./request.js";
-export type { GenerationPortSubset, GenerationRequest } from "./request.js";
+export type {
+ GenerationMediaBinding,
+ GenerationPortSubset,
+ GenerationRequest,
+ GenerationRequestDraft,
+} from "./request.js";
export {
generatedImageSetSchema,
generatedAudioSetSchema,
diff --git a/packages/generation/src/request.ts b/packages/generation/src/request.ts
index 02c7a7db..1d1ee216 100644
--- a/packages/generation/src/request.ts
+++ b/packages/generation/src/request.ts
@@ -1,4 +1,5 @@
-import type { ObjectFieldSchema, ValueSchema } from "@narratage/protocol";
+import { canonicalize } from "@narratage/protocol";
+import type { BlobRef, ObjectFieldSchema, ValueSchema } from "@narratage/protocol";
import { assertGenerationBlobRef, sealGenerationRequest } from "./identity.js";
import { generationBlobRefSchema, generationObjectSchema } from "./schema.js";
@@ -19,6 +20,8 @@ import type {
* mechanism serves every model and every service reselling it.
*/
export const GENERATION_REQUEST_V1 = "svml.generation-request@1" as const;
+export const GENERATION_REQUEST_DRAFT_V1 = "svml.generation-request-draft@1" as const;
+export const GENERATION_MEDIA_BINDING_V1 = "svml.generation-media-binding@1" as const;
export type GenerationRequest = {
readonly contract: "svml.generation-request@1";
@@ -27,6 +30,24 @@ export type GenerationRequest = {
readonly ports: Readonly<Record<string, readonly GenerationPortValue[]>>;
};
+/**
+ * A request while graph inputs are still being attached. It is deliberately a
+ * different contract from GenerationRequest: Providers can only receive the
+ * finalized request after the exact model package has checked every port rule.
+ */
+export type GenerationRequestDraft = {
+ readonly contract: "svml.generation-request-draft@1";
+ readonly model: string;
+ readonly ports: Readonly<Record<string, readonly GenerationPortValue[]>>;
+};
+
+/** Authored instructions for attaching one graph Blob edge to one media port. */
+export type GenerationMediaBinding = {
+ readonly contract: "svml.generation-media-binding@1";
+ readonly role: GenerationMediaRole;
+ readonly fields?: Readonly<Record<string, string | number | boolean>>;
+};
+
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
@@ -68,6 +89,16 @@ function itemFieldsSchema(fields: readonly GenerationPortItemField[]): ValueSche
} satisfies ObjectFieldSchema])));
}
+/** Schema for the non-artifact half of one media-port attachment. */
+export function mediaBindingSchemaFromPort(port: GenerationMediaPort): ValueSchema {
+ const fields = port.value.itemFields ?? [];
+ return generationObjectSchema({
+ contract: { schema: { kind: "literal", value: GENERATION_MEDIA_BINDING_V1 } },
+ role: { schema: { kind: "string", enum: [...port.value.accepts] } },
+ ...(fields.length === 0 ? {} : { fields: { schema: itemFieldsSchema(fields) } }),
+ });
+}
+
function portItemSchema(port: GenerationPort): ValueSchema {
if (!isMediaPort(port)) return scalarSchema(port.value);
const media = port.value;
@@ -87,6 +118,12 @@ export type GenerationPortSubset = {
* the missing fact exists.
*/
readonly omit?: readonly string[];
+ /**
+ * Ports that may be absent while a graph is still attaching values. If they
+ * are present, their item and cardinality rules are still checked. Exact
+ * finalization never uses this escape hatch.
+ */
+ readonly defer?: readonly string[];
};
/** Derive the Schema of the port map alone, optionally leaving some ports for later. */
@@ -96,8 +133,13 @@ export function portsObjectSchema(
): ValueSchema {
assertGenerationPortTable(table);
const omit = new Set(options.omit ?? []);
+ const defer = new Set(options.defer ?? []);
omit.forEach((name) => assert(table.ports.some((port) => port.name === name),
`${table.model} cannot omit undeclared port ${name}`));
+ defer.forEach((name) => {
+ assert(table.ports.some((port) => port.name === name), `${table.model} cannot defer undeclared port ${name}`);
+ assert(!omit.has(name), `${table.model} cannot both omit and defer port ${name}`);
+ });
return generationObjectSchema(Object.fromEntries(table.ports
.filter((port) => !omit.has(port.name))
.map((port) => [port.name, {
@@ -107,7 +149,7 @@ export function portsObjectSchema(
minItems: Math.max(1, port.minItems),
maxItems: port.maxItems,
},
- ...(port.minItems === 0 ? { optional: true } : {}),
+ ...(port.minItems === 0 || defer.has(port.name) ? { optional: true } : {}),
} satisfies ObjectFieldSchema])));
}
@@ -120,6 +162,16 @@ export function requestSchemaFromPorts(table: GenerationPortTable): ValueSchema
});
}
+/** Structural schema for the model-owned request draft used by graph assembly. */
+export function requestDraftSchemaFromPorts(table: GenerationPortTable): ValueSchema {
+ const media = table.ports.filter(isMediaPort).map((port) => port.name);
+ return generationObjectSchema({
+ contract: { schema: { kind: "literal", value: GENERATION_REQUEST_DRAFT_V1 } },
+ model: { schema: { kind: "literal", value: table.model } },
+ ports: { schema: portsObjectSchema(table, { defer: media }) },
+ });
+}
+
function verifyScalar(value: unknown, kind: GenerationPortScalarKind, subject: string): void {
if (kind.kind === "text") {
assert(typeof value === "string" && value.length > 0, `${subject} must be non-empty text`);
@@ -150,19 +202,17 @@ function verifyScalar(value: unknown, kind: GenerationPortScalarKind, subject: s
);
}
-function verifyMediaValue(value: unknown, port: GenerationMediaPort, subject: string): void {
+function verifyMediaItemFields(value: Record<string, unknown>, port: GenerationMediaPort, subject: string): void {
const media = port.value;
- const item = plainObject(value, subject);
- const role = item.role;
+ const role = value.role;
assert(typeof role === "string" && media.accepts.includes(role as GenerationMediaRole),
`${subject} role must be one of ${media.accepts.join(", ")}`);
- assertGenerationBlobRef(item.artifact, mediaPrefix(role as GenerationMediaRole));
const declared = media.itemFields ?? [];
if (declared.length === 0) {
- assert(item.fields === undefined, `${subject} declares no item fields`);
+ assert(value.fields === undefined, `${subject} declares no item fields`);
return;
}
- const fields = plainObject(item.fields ?? {}, `${subject} fields`);
+ const fields = plainObject(value.fields ?? {}, `${subject} fields`);
const known = new Set(declared.map((field) => field.name));
Object.keys(fields).forEach((name) => assert(known.has(name), `${subject} has unknown item field ${name}`));
for (const field of declared) {
@@ -183,6 +233,12 @@ function verifyMediaValue(value: unknown, port: GenerationMediaPort, subject: st
}
}
+function verifyMediaValue(value: unknown, port: GenerationMediaPort, subject: string): void {
+ const item = plainObject(value, subject);
+ verifyMediaItemFields(item, port, subject);
+ assertGenerationBlobRef(item.artifact, mediaPrefix(item.role as GenerationMediaRole));
+}
+
/**
* Full standalone verification against the port table. Core's generic schema
* admission runs first; this adds the facts a Schema cannot state, such as the
@@ -195,6 +251,7 @@ export function verifyPortsAgainstTable(
): void {
assertGenerationPortTable(table);
const omit = new Set(options.omit ?? []);
+ const defer = new Set(options.defer ?? []);
const ports = plainObject(value, `${table.model} ports`);
const declared = new Map(table.ports.map((port) => [port.name, port]));
Object.keys(ports).forEach((name) => {
@@ -208,7 +265,7 @@ export function verifyPortsAgainstTable(
const subject = `${table.model} port ${port.name}`;
const supplied = ports[port.name];
if (supplied === undefined) {
- assert(port.minItems === 0, `${subject} is required`);
+ assert(port.minItems === 0 || defer.has(port.name), `${subject} is required`);
continue;
}
assert(Array.isArray(supplied), `${subject} must be an array`);
@@ -232,7 +289,7 @@ export function verifyPortsAgainstTable(
}
if (requirement.kind === "requiresAnyOf") {
if (!present.has(requirement.port)) continue;
- const satisfied = requirement.anyOf.some((name) => present.has(name) || omit.has(name));
+ const satisfied = requirement.anyOf.some((name) => present.has(name) || omit.has(name) || defer.has(name));
assert(satisfied,
`${table.model} port ${requirement.port} needs at least one of ${requirement.anyOf.join(", ")}`);
continue;
@@ -247,12 +304,94 @@ export function verifyPortsAgainstTable(
continue;
}
if (!present.has(requirement.port)) continue;
- const missing = requirement.needs.filter((name) => !present.has(name) && !omit.has(name));
+ const missing = requirement.needs.filter((name) => !present.has(name) && !omit.has(name) && !defer.has(name));
assert(missing.length === 0,
`${table.model} port ${requirement.port} also requires ${missing.join(", ")}`);
}
}
+function mediaPorts(table: GenerationPortTable): readonly GenerationMediaPort[] {
+ return table.ports.filter(isMediaPort);
+}
+
+export function verifyRequestDraftAgainstPorts(
+ table: GenerationPortTable,
+ value: unknown,
+): asserts value is GenerationRequestDraft {
+ const draft = plainObject(value, `${table.model} request draft`);
+ assert(draft.contract === GENERATION_REQUEST_DRAFT_V1, `${table.model} request draft contract is invalid`);
+ assert(draft.model === table.model, `${table.model} request draft model is invalid`);
+ verifyPortsAgainstTable(table, draft.ports, { defer: mediaPorts(table).map((port) => port.name) });
+}
+
+export function sealGenerationRequestDraft(
+ table: GenerationPortTable,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+): GenerationRequestDraft {
+ const draft = canonicalize({
+ contract: GENERATION_REQUEST_DRAFT_V1,
+ model: table.model,
+ ports: Object.fromEntries(Object.entries(ports).filter(([, values]) => values.length > 0)),
+ }) as unknown as GenerationRequestDraft;
+ verifyRequestDraftAgainstPorts(table, draft);
+ return draft;
+}
+
+export function verifyGenerationMediaBinding(
+ port: GenerationMediaPort,
+ value: unknown,
+): asserts value is GenerationMediaBinding {
+ const binding = plainObject(value, `${port.name} media binding`);
+ assert(binding.contract === GENERATION_MEDIA_BINDING_V1, `${port.name} media binding contract is invalid`);
+ verifyMediaItemFields(binding, port, `${port.name} media binding`);
+}
+
+export function sealGenerationMediaBinding(
+ port: GenerationMediaPort,
+ value: Omit<GenerationMediaBinding, "contract">,
+): GenerationMediaBinding {
+ const binding = canonicalize({ contract: GENERATION_MEDIA_BINDING_V1, ...value }) as unknown as GenerationMediaBinding;
+ verifyGenerationMediaBinding(port, binding);
+ return binding;
+}
+
+/**
+ * Attach one Blob supplied by a real graph edge. The returned draft contains
+ * the Provider-facing value, while Derivation preserves where that Blob came
+ * from; no lineage or project metadata is copied through the payload.
+ */
+export function bindGenerationMedia(
+ table: GenerationPortTable,
+ draft: GenerationRequestDraft,
+ portName: string,
+ binding: GenerationMediaBinding,
+ artifact: BlobRef,
+): GenerationRequestDraft {
+ verifyRequestDraftAgainstPorts(table, draft);
+ const port = table.ports.find((candidate): candidate is GenerationMediaPort =>
+ candidate.name === portName && isMediaPort(candidate));
+ assert(port !== undefined, `${table.model} has no media port ${portName}`);
+ verifyGenerationMediaBinding(port, binding);
+ const value = {
+ role: binding.role,
+ artifact,
+ ...(binding.fields === undefined ? {} : { fields: binding.fields }),
+ };
+ verifyMediaValue(value, port, `${table.model} port ${portName}`);
+ return sealGenerationRequestDraft(table, {
+ ...draft.ports,
+ [portName]: [...(draft.ports[portName] ?? []), value],
+ });
+}
+
+export function finalizeGenerationRequestDraft(
+ table: GenerationPortTable,
+ draft: GenerationRequestDraft,
+): GenerationRequest {
+ verifyRequestDraftAgainstPorts(table, draft);
+ return sealGenerationPortRequest(table, draft.ports);
+}
+
export function verifyRequestAgainstPorts(
table: GenerationPortTable,
value: unknown,
diff --git a/packages/generation/test/ports.test.ts b/packages/generation/test/ports.test.ts
index d06b621b..62e52fc5 100644
--- a/packages/generation/test/ports.test.ts
+++ b/packages/generation/test/ports.test.ts
@@ -4,9 +4,13 @@ import test from "node:test";
import { MemoryArtifactStore } from "@narratage/driver-node";
import {
assertMappingCoversPorts,
+ bindGenerationMedia,
compileWireRequest,
+ finalizeGenerationRequestDraft,
requestSchemaFromPorts,
+ sealGenerationMediaBinding,
sealGenerationPortRequest,
+ sealGenerationRequestDraft,
sealGenerationPortTable,
selectWireModel,
verifyRequestAgainstPorts,
@@ -202,3 +206,36 @@ test("coverage rejects a mapping that forgets a port the model declares", () =>
};
assert.throws(() => assertMappingCoversPorts(table, wrongShape), /must use valueArray/u);
});
+
+test("runtime media stays on graph edges until a model-owned draft is finalized", async () => {
+ const { image, video, audio } = await artifacts();
+ const base = sealGenerationRequestDraft(table, {
+ prompt: ["hello"], duration: [8], resolution: ["720p"],
+ });
+ const audioPort = table.ports.find((port) => port.name === "referenceAudio");
+ const imagePort = table.ports.find((port) => port.name === "referenceImage");
+ const excerptPort = table.ports.find((port) => port.name === "excerpt");
+ assert.ok(audioPort?.value.kind === "media");
+ assert.ok(imagePort?.value.kind === "media");
+ assert.ok(excerptPort?.value.kind === "media");
+
+ // An intermediate draft may receive audio first; the exact cross-port rule
+ // is enforced only once all explicitly connected edges have been attached.
+ const withAudio = bindGenerationMedia(table, base, "referenceAudio",
+ sealGenerationMediaBinding(audioPort as never, { role: "audio" }), audio);
+ assert.throws(() => finalizeGenerationRequestDraft(table, withAudio), /needs at least one of referenceImage/u);
+ const withImage = bindGenerationMedia(table, withAudio, "referenceImage",
+ sealGenerationMediaBinding(imagePort as never, { role: "image" }), image);
+ const withExcerpt = bindGenerationMedia(table, withImage, "excerpt",
+ sealGenerationMediaBinding(excerptPort as never, {
+ role: "video", fields: { startSec: 1, endSec: 3 },
+ }), video);
+ const exact = finalizeGenerationRequestDraft(table, withExcerpt);
+ assert.equal(exact.contract, "svml.generation-request@1");
+ assert.deepEqual(exact.ports.referenceAudio?.[0], { role: "audio", artifact: audio });
+ assert.deepEqual(exact.ports.excerpt?.[0], {
+ role: "video", artifact: video, fields: { startSec: 1, endSec: 3 },
+ });
+ assert.throws(() => bindGenerationMedia(table, base, "referenceImage",
+ sealGenerationMediaBinding(imagePort as never, { role: "image" }), audio), /image\//u);
+});
diff --git a/packages/gpt-image/README.md b/packages/gpt-image/README.md
index 9c193a71..9f9c1066 100644
--- a/packages/gpt-image/README.md
+++ b/packages/gpt-image/README.md
@@ -1,12 +1,19 @@
# `@narratage/gpt-image`
-Exact author/compute contracts for GPT Image 2 text-to-image and image-to-image requests.
+Exact model/compute contracts for GPT Image 2 requests.
-The two modes are separate graph endpoints with mode-specific validation and a common
-provider-neutral `GeneratedImageSet` result. The package owns model semantics but no API key,
-Provider selection, queue or network code. `@narratage/provider-kie` is one optional Runtime
-implementation.
+One closed Port Table accepts the supported scalar settings and optional reference images. Runtime-
+produced references remain explicit Blob edges: the model-owned Draft is bound one edge at a time,
+then finalized into the only `GenerationRequest` a Provider can receive. The package owns model
+semantics but no API key, Provider selection, queue or network code. `@narratage/provider-kie` is
+one optional Runtime implementation.
-The installed package is directly activatable by the SVML package loader. It currently exposes exact
-compute Fragments rather than a high-level author Surface; an author kit may wrap those Fragments without
-changing the model or Provider contract.
+The one physical package exposes two independently importable logical modules:
+
+- `@narratage/gpt-image@1`: the raw exact model;
+- `@narratage/gpt-image/clean@1`: generation followed by the existing explicit image-transform
+ Program, exporting one cleaned image while retaining both operations in the graph.
+
+It currently exposes low-level Draft/Fragment factories rather than a high-level Prompt Surface.
+Prompt authoring and assembly are intentionally deferred to the separate Prompt redesign; that work
+will not change the media edges, exact request or Provider contract described here.
diff --git a/packages/gpt-image/package.json b/packages/gpt-image/package.json
index 8da293e2..e6d20ffe 100644
--- a/packages/gpt-image/package.json
+++ b/packages/gpt-image/package.json
@@ -10,7 +10,10 @@
"activation": "./src/activation.ts"
},
"dependencies": {
+ "@narratage/artifact": "workspace:*",
+ "@narratage/elaborator": "workspace:*",
"@narratage/generation": "workspace:*",
+ "@narratage/image-transform": "workspace:*",
"@narratage/model-kit": "workspace:*",
"@narratage/protocol": "workspace:*"
}
diff --git a/packages/gpt-image/src/activation.ts b/packages/gpt-image/src/activation.ts
index 4cf15860..4282d930 100644
--- a/packages/gpt-image/src/activation.ts
+++ b/packages/gpt-image/src/activation.ts
@@ -1,3 +1,14 @@
-import { gptImageComponent, gptImageManifest } from "./index.js";
-export const svmlPackage = { format: "svml.node-package@1" as const, name: "@narratage/gpt-image", modules: [{ manifest: gptImageManifest, specifiers: ["@narratage/gpt-image", "@narratage/gpt-image@1"] }], components: [gptImageComponent] };
+import { gptImageCleanManifest, gptImageComponent, gptImageManifest } from "./index.js";
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/gpt-image",
+ modules: [{
+ manifest: gptImageManifest,
+ specifiers: ["@narratage/gpt-image", "@narratage/gpt-image@1"],
+ }, {
+ manifest: gptImageCleanManifest,
+ specifiers: ["@narratage/gpt-image/clean", "@narratage/gpt-image/clean@1"],
+ }],
+ components: [gptImageComponent],
+};
export default svmlPackage;
diff --git a/packages/gpt-image/src/fragment.ts b/packages/gpt-image/src/fragment.ts
new file mode 100644
index 00000000..755f979d
--- /dev/null
+++ b/packages/gpt-image/src/fragment.ts
@@ -0,0 +1,85 @@
+import { artifactTypes } from "@narratage/artifact";
+import { sealGraphFragment } from "@narratage/elaborator";
+import { generationProducers } from "@narratage/generation";
+import { imageTransformProducers, imageTransformTypes } from "@narratage/image-transform";
+import { exactModelMediaInputNames } from "@narratage/model-kit";
+import type { ExactModelMediaInput } from "@narratage/model-kit";
+import type { FragmentOperation } from "@narratage/elaborator";
+
+import { gptImageEndpoints } from "./index.js";
+
+const input = (name: string) => ({ kind: "fragment-input" as const, name });
+const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+
+/**
+ * Official clean GPT Image result: exact generation followed by the explicit,
+ * replaceable Raster transform already used everywhere else. This is one
+ * author-facing component but remains two visible graph operations/Needs.
+ */
+export function createGptImageCleanFragment(
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+) {
+ const endpoint = gptImageEndpoints.image!;
+ const names = mediaInputs.map((item) => item.name);
+ if (new Set(names).size !== names.length) throw new Error("GPT Image media input names must be unique");
+ const inputs = [
+ { name: "draft", type: endpoint.draftType },
+ { name: "cleanup", type: imageTransformTypes.program },
+ ];
+ const operations: FragmentOperation[] = [];
+ let draft = input("draft") as ReturnType<typeof input> | ReturnType<typeof operation>;
+ for (const [index, item] of mediaInputs.entries()) {
+ const binding = endpoint.mediaBindings[item.port];
+ if (binding === undefined) throw new Error(`gpt-image-2 has no media port ${item.port}`);
+ const inputNames = exactModelMediaInputNames(item.name);
+ inputs.push({ name: inputNames.binding, type: binding.type });
+ inputs.push({ name: inputNames.artifact, type: artifactTypes.blob });
+ const id = `bind:${String(index + 1).padStart(4, "0")}:${item.port}`;
+ operations.push({
+ id,
+ producer: binding.producer,
+ inputs: { draft, binding: input(inputNames.binding), artifact: input(inputNames.artifact) },
+ result: { kind: "output", name: "draft" },
+ });
+ draft = operation(id);
+ }
+ operations.push(
+ {
+ id: "finalize-request",
+ producer: endpoint.finalizeProducer,
+ inputs: { draft },
+ result: { kind: "output", name: "request" },
+ },
+ {
+ id: "generate",
+ producer: endpoint.producer,
+ inputs: { request: operation("finalize-request") },
+ result: { kind: "need", name: "generation", accepts: "exact" },
+ },
+ {
+ id: "select-primary-image",
+ producer: generationProducers.primaryImage,
+ inputs: { set: operation("generate") },
+ result: { kind: "output", name: "image" },
+ },
+ {
+ id: "clean-image",
+ producer: imageTransformProducers.request,
+ inputs: { source: operation("select-primary-image"), program: input("cleanup") },
+ result: { kind: "need", name: "image", accepts: "exact" },
+ },
+ );
+ const shape = mediaInputs.map((item) => `${item.name}=${item.port}`).join(",") || "no-media";
+ return sealGraphFragment({
+ name: `@narratage/gpt-image/clean[${shape}]@1`,
+ inputs,
+ operations,
+ exports: [{
+ name: "image",
+ type: artifactTypes.blob,
+ root: operation("clean-image"),
+ semanticInputs: inputs.map((entry) => entry.name),
+ fidelity: "exact",
+ }],
+ });
+}
diff --git a/packages/gpt-image/src/index.ts b/packages/gpt-image/src/index.ts
index 345db8ee..dc092335 100644
--- a/packages/gpt-image/src/index.ts
+++ b/packages/gpt-image/src/index.ts
@@ -1,6 +1,15 @@
-import { sealGenerationPortRequest, sealGenerationPortTable } from "@narratage/generation";
+import {
+ sealGenerationPortRequest,
+ sealGenerationRequestDraft,
+ sealGenerationPortTable,
+} from "@narratage/generation";
import type { GenerationPortTable, GenerationPortValue, GenerationRequest } from "@narratage/generation";
import { defineExactModelModule } from "@narratage/model-kit";
+import { digestOf } from "@narratage/protocol";
+import {
+ imageTransformManifestDigest,
+ imageTransformModuleRef,
+} from "@narratage/image-transform";
export const gptImageModuleRef = { name: "@narratage/gpt-image", version: "1" } as const;
@@ -32,6 +41,13 @@ export function sealGptImage2Request(
return sealGenerationPortRequest(gptImage2Ports, ports);
}
+/** Scalar request seed; reference images are attached later by explicit graph edges. */
+export function sealGptImage2Draft(
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+) {
+ return sealGenerationRequestDraft(gptImage2Ports, ports);
+}
+
export const gptImageDefinition = defineExactModelModule({
module: gptImageModuleRef,
endpoints: [{
@@ -46,3 +62,21 @@ export const gptImageManifest = gptImageDefinition.manifest;
export const gptImageManifestDigest = gptImageDefinition.manifestDigest;
export const gptImageEndpoints = gptImageDefinition.endpoints;
export const gptImageComponent = gptImageDefinition.component;
+
+/** Optional authoring submodule; the exact GPT model remains independent of post-processing. */
+export const gptImageCleanModuleRef = { name: "@narratage/gpt-image/clean", version: "1" } as const;
+export const gptImageCleanManifest = {
+ format: "svml.module@1" as const,
+ name: gptImageCleanModuleRef.name,
+ version: gptImageCleanModuleRef.version,
+ dependencies: [
+ { module: gptImageModuleRef, digest: gptImageManifestDigest },
+ { module: imageTransformModuleRef, digest: imageTransformManifestDigest },
+ ],
+ types: [],
+ capabilities: [],
+ surfaces: [],
+ producers: [],
+};
+export const gptImageCleanManifestDigest = digestOf(gptImageCleanManifest);
+export { createGptImageCleanFragment } from "./fragment.js";
diff --git a/packages/gpt-image/test/broll-graph.test.ts b/packages/gpt-image/test/broll-graph.test.ts
new file mode 100644
index 00000000..b40bc12e
--- /dev/null
+++ b/packages/gpt-image/test/broll-graph.test.ts
@@ -0,0 +1,244 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { artifactManifest, artifactTypes } from "@narratage/artifact";
+import {
+ createResolvedClosure,
+ digestOf,
+ link,
+ sealBuildRequest,
+ sealRecord,
+ sealTypeValidationReceipt,
+ sealTypedModule,
+ start,
+} from "@narratage/core";
+import { elaborateAuthorModule, sealAuthorModule } from "@narratage/elaborator";
+import {
+ generationManifest,
+ sealGenerationMediaBinding,
+ sealGenerationRequestDraft,
+} from "@narratage/generation";
+import {
+ createGptImageCleanFragment,
+ gptImage2Ports,
+ gptImageCleanManifest,
+ gptImageEndpoints,
+ gptImageManifest,
+ sealGptImage2Draft,
+} from "@narratage/gpt-image";
+import {
+ gptImageDenoiseV1,
+ imageTransformImplementationDigests,
+ imageTransformManifest,
+ imageTransformTypes,
+} from "@narratage/image-transform";
+import { exactModelMediaInputNames } from "@narratage/model-kit";
+import { mediaManifest } from "@narratage/media";
+import { narrativeManifest } from "@narratage/narrative";
+import { programSpaceManifest } from "@narratage/program-space";
+import type { CanonicalValue, Digest, ModuleManifest, StoredValue, TypeRef } from "@narratage/protocol";
+import { rasterManifest } from "@narratage/raster";
+import { createProvidedCandidate, resolveRealization, sealRealizationOverlay } from "@narratage/run";
+import {
+ createSeedanceAssembledGenerationFragment,
+ seedanceEndpoints,
+ seedanceManifest,
+ seedancePorts,
+} from "@narratage/seedance";
+import { speechManifest } from "@narratage/speech";
+import { spatialManifest } from "@narratage/spatial";
+
+const origin = {
+ kind: "authored" as const,
+ sourceDigest: digestOf("source:broll-topology"),
+ frontendClosureDigest: digestOf("frontend:broll-topology"),
+};
+
+const image = (name: string) => ({
+ kind: "blob" as const,
+ digest: digestOf(`image:${name}`),
+ size: 100,
+ mediaType: "image/png",
+});
+
+function fixture() {
+ const manifests: ModuleManifest[] = [
+ artifactManifest,
+ generationManifest,
+ rasterManifest,
+ imageTransformManifest,
+ gptImageManifest,
+ gptImageCleanManifest,
+ narrativeManifest,
+ mediaManifest,
+ programSpaceManifest,
+ spatialManifest,
+ speechManifest,
+ seedanceManifest,
+ ];
+ const closure = createResolvedClosure(manifests);
+ const gpt = gptImageEndpoints.image!;
+ const seedance = seedanceEndpoints.mini!;
+ const gptBindingPort = gptImage2Ports.ports.find((port) => port.name === "images");
+ const seedanceBindingPort = seedancePorts["seedance-2-mini"].ports.find((port) => port.name === "referenceImage");
+ assert.ok(gptBindingPort?.value.kind === "media");
+ assert.ok(seedanceBindingPort?.value.kind === "media");
+ const records: Array<{
+ readonly id: string;
+ readonly type: TypeRef;
+ readonly value: StoredValue;
+ readonly validatorDigest?: Digest;
+ }> = [];
+ const add = (id: string, type: TypeRef, value: StoredValue, validatorDigest?: Digest) =>
+ records.push({ id, type, value, ...(validatorDigest === undefined ? {} : { validatorDigest }) });
+ add("person", artifactTypes.blob, image("person"));
+ add("product", artifactTypes.blob, image("product"));
+ add("cleanup", imageTransformTypes.program, {
+ kind: "inline", value: gptImageDenoiseV1 as unknown as CanonicalValue,
+ }, imageTransformImplementationDigests.validator);
+
+ const gptFragments = {
+ holding: createGptImageCleanFragment([
+ { name: "person", port: "images" }, { name: "product", port: "images" },
+ ]),
+ walking: createGptImageCleanFragment([{ name: "holding", port: "images" }]),
+ interview: createGptImageCleanFragment([{ name: "holding", port: "images" }]),
+ };
+ const addGptInputs = (id: string, names: readonly string[]) => {
+ add(`${id}.draft`, gpt.draftType, {
+ kind: "inline",
+ value: sealGptImage2Draft({
+ prompt: [`${id} prompt`], aspectRatio: ["9:16"], resolution: ["1K"],
+ }) as unknown as CanonicalValue,
+ }, gpt.draftValidatorDigest);
+ for (const name of names) add(`${id}.${name}.binding`, gpt.mediaBindings.images!.type, {
+ kind: "inline",
+ value: sealGenerationMediaBinding(gptBindingPort as never, { role: "image" }) as unknown as CanonicalValue,
+ });
+ };
+ addGptInputs("holding", ["person", "product"]);
+ addGptInputs("walking", ["holding"]);
+ addGptInputs("interview", ["holding"]);
+
+ const seedanceMedia = ["holding", "walking", "interview"].map((name) => ({ name, port: "referenceImage" }));
+ const seedanceFragment = createSeedanceAssembledGenerationFragment(seedance, seedanceMedia);
+ add("montage.draft", seedance.draftType, {
+ kind: "inline",
+ value: sealGenerationRequestDraft(seedancePorts["seedance-2-mini"], {
+ prompt: ["A fast montage from all three reference images."],
+ resolution: ["720p"], aspectRatio: ["9:16"], duration: [6], generateAudio: [false], webSearch: [false],
+ }) as unknown as CanonicalValue,
+ }, seedance.draftValidatorDigest);
+ for (const name of ["holding", "walking", "interview"]) add(`montage.${name}.binding`, seedance.mediaBindings.referenceImage!.type, {
+ kind: "inline",
+ value: sealGenerationMediaBinding(seedanceBindingPort as never, { role: "image" }) as unknown as CanonicalValue,
+ });
+
+ const componentInputs = (
+ id: string,
+ media: readonly { readonly name: string; readonly source: { readonly kind: "record"; readonly id: string } | { readonly kind: "component-output"; readonly component: string; readonly output: string } }[],
+ ) => Object.fromEntries([
+ ["draft", { kind: "record" as const, id: `${id}.draft` }],
+ ["cleanup", { kind: "record" as const, id: "cleanup" }],
+ ...media.flatMap((item) => {
+ const names = exactModelMediaInputNames(item.name);
+ return [
+ [names.binding, { kind: "record" as const, id: `${id}.${item.name}.binding` }],
+ [names.artifact, item.source],
+ ] as const;
+ }),
+ ]);
+ const components = [{
+ id: "holding",
+ fragment: gptFragments.holding.id,
+ inputs: componentInputs("holding", [
+ { name: "person", source: { kind: "record", id: "person" } },
+ { name: "product", source: { kind: "record", id: "product" } },
+ ]),
+ outputs: { image: "holding.image" },
+ }, {
+ id: "walking",
+ fragment: gptFragments.walking.id,
+ inputs: componentInputs("walking", [{
+ name: "holding", source: { kind: "component-output", component: "holding", output: "image" },
+ }]),
+ outputs: { image: "walking.image" },
+ }, {
+ id: "interview",
+ fragment: gptFragments.interview.id,
+ inputs: componentInputs("interview", [{
+ name: "holding", source: { kind: "component-output", component: "holding", output: "image" },
+ }]),
+ outputs: { image: "interview.image" },
+ }, {
+ id: "montage",
+ fragment: seedanceFragment.id,
+ inputs: Object.fromEntries([
+ ["draft", { kind: "record" as const, id: "montage.draft" }],
+ ...["holding", "walking", "interview"].flatMap((name) => {
+ const names = exactModelMediaInputNames(name);
+ return [
+ [names.binding, { kind: "record" as const, id: `montage.${name}.binding` }],
+ [names.artifact, { kind: "component-output" as const, component: name, output: "image" }],
+ ] as const;
+ }),
+ ]),
+ outputs: { video: "montage.video" },
+ }];
+ const typed = sealTypedModule({
+ id: "broll-records",
+ closureDigest: closure.digest,
+ records: records.map(({ validatorDigest, ...record }) => {
+ const sealed = sealRecord({ ...record, conformance: "exact", origin });
+ return validatorDigest === undefined ? sealed : {
+ ...sealed,
+ validation: sealTypeValidationReceipt({
+ type: sealed.type,
+ recordDigest: sealed.digest,
+ validatorDigest,
+ }),
+ };
+ }),
+ });
+ const program = link(closure, [typed]);
+ const fragments = new Map([
+ ...Object.values(gptFragments).map((fragment) => [fragment.id, fragment] as const),
+ [seedanceFragment.id, seedanceFragment] as const,
+ ]);
+ const elaborated = elaborateAuthorModule(program, sealAuthorModule({ name: "broll", components }),
+ (id) => fragments.get(id));
+ return { program, graph: elaborated.graph };
+}
+
+test("the common B-roll topology is one graph with one shared holding image", () => {
+ const { program, graph } = fixture();
+ const state = start(program, graph, sealBuildRequest({
+ graph: graph.id,
+ targets: [{ output: "montage.video", accepts: "exact" }],
+ satisfactions: [],
+ }));
+ const producers = state.plan.steps.map((step) => step.producer.name);
+ assert.equal(producers.filter((name) => name === "request-gpt-image-2").length, 3);
+ assert.equal(producers.filter((name) => name === "request-image-transform").length, 3);
+ assert.equal(producers.filter((name) => name === "request-seedance-2-mini").length, 1);
+ const holding = graph.outputs.find((output) => output.id === "holding.image");
+ assert.ok(holding);
+ assert.equal(graph.operations.filter((operation) =>
+ Object.values(operation.inputs).some((input) => input.kind === "logical-output" && input.id === "holding.image")).length, 3);
+});
+
+test("an explicitly selected holding-image Candidate prunes only that branch", () => {
+ const { program, graph } = fixture();
+ const candidate = createProvidedCandidate({ type: artifactTypes.blob, value: image("approved-holding") });
+ const overlay = sealRealizationOverlay({ sourceGraph: graph.id, candidates: [candidate], operations: [] });
+ const realized = resolveRealization(program, graph, [overlay]);
+ const state = start(program, realized.graph, sealBuildRequest({
+ graph: realized.graph.id,
+ targets: [{ output: "montage.video", accepts: "exact" }],
+ satisfactions: [{ output: "holding.image", candidate: candidate.id, fidelity: "exact" }],
+ }));
+ const producers = state.plan.steps.map((step) => step.producer.name);
+ assert.equal(producers.filter((name) => name === "request-gpt-image-2").length, 2);
+ assert.equal(producers.filter((name) => name === "request-image-transform").length, 2);
+ assert.equal(producers.filter((name) => name === "request-seedance-2-mini").length, 1);
+});
diff --git a/packages/mimo-tts/src/fragment.ts b/packages/mimo-tts/src/fragment.ts
index bb2734dd..281ad31d 100644
--- a/packages/mimo-tts/src/fragment.ts
+++ b/packages/mimo-tts/src/fragment.ts
@@ -1,36 +1,10 @@
-import { artifactTypes } from "@narratage/artifact";
-import { sealGraphFragment } from "@narratage/elaborator";
-import { generationProducers } from "@narratage/generation";
-import type { ExactModelEndpoint } from "@narratage/model-kit";
+import { createExactModelPrimaryGenerationFragment } from "@narratage/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput } from "@narratage/model-kit";
-const input = (name: string) => ({ kind: "fragment-input" as const, name });
-const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
-
-/** The model result stays atomic until this deterministic projection exposes one ordinary audio Blob. */
-export function createMimoTtsAudioFragment(endpoint: ExactModelEndpoint) {
- return sealGraphFragment({
- name: `@narratage/mimo-tts/${endpoint.key}-primary-audio@1`,
- inputs: [{ name: "request", type: endpoint.requestType }],
- operations: [
- {
- id: "generate",
- producer: endpoint.producer,
- inputs: { request: input("request") },
- result: { kind: "need", name: "generation", accepts: "exact" },
- },
- {
- id: "select-primary-audio",
- producer: generationProducers.primaryAudio,
- inputs: { set: operation("generate") },
- result: { kind: "output", name: "audio" },
- },
- ],
- exports: [{
- name: "audio",
- type: artifactTypes.blob,
- root: operation("select-primary-audio"),
- semanticInputs: ["request"],
- fidelity: "exact",
- }],
- });
+/** Model result projection over the same graph-native request assembly as every exact model. */
+export function createMimoTtsAudioFragment(
+ endpoint: ExactModelEndpoint,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+) {
+ return createExactModelPrimaryGenerationFragment(endpoint, mediaInputs);
}
diff --git a/packages/mimo-tts/src/index.ts b/packages/mimo-tts/src/index.ts
index f74cde4e..d685387d 100644
--- a/packages/mimo-tts/src/index.ts
+++ b/packages/mimo-tts/src/index.ts
@@ -1,5 +1,4 @@
-import { artifactDependency } from "@narratage/artifact";
-import { sealGenerationPortRequest, sealGenerationPortTable } from "@narratage/generation";
+import { sealGenerationPortRequest, sealGenerationRequestDraft, sealGenerationPortTable } from "@narratage/generation";
import type { GenerationPortTable, GenerationPortValue, GenerationRequest } from "@narratage/generation";
import { defineExactModelModule } from "@narratage/model-kit";
import { narrativeDependency } from "@narratage/narrative";
@@ -70,6 +69,13 @@ export function sealMimoTtsRequest(
return sealGenerationPortRequest(mimoTtsPorts[model], ports);
}
+export function sealMimoTtsRequestDraft(
+ model: MimoTtsModel,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+) {
+ return sealGenerationRequestDraft(mimoTtsPorts[model], ports);
+}
+
const base = defineExactModelModule({
module: mimoTtsModuleRef,
endpoints: ([
@@ -93,11 +99,11 @@ export const mimoTtsSurfaceDigests = {
export const mimoTtsManifest = {
...base.manifest,
- dependencies: [...base.manifest.dependencies, artifactDependency, narrativeDependency],
+ dependencies: [...base.manifest.dependencies, narrativeDependency],
surfaces: [
{
name: "preset", tag: "Preset", mode: "structured",
- outputs: [mimoTtsEndpoints.preset.requestType],
+ outputs: [mimoTtsEndpoints.preset.draftType],
implementation: {
kind: "trusted-frontend-surface" as const,
locator: "@narratage/mimo-tts/preset-surface",
@@ -106,7 +112,7 @@ export const mimoTtsManifest = {
},
{
name: "voiceDesign", tag: "VoiceDesign", mode: "structured",
- outputs: [mimoTtsEndpoints.voiceDesign.requestType],
+ outputs: [mimoTtsEndpoints.voiceDesign.draftType],
implementation: {
kind: "trusted-frontend-surface" as const,
locator: "@narratage/mimo-tts/voice-design-surface",
@@ -115,7 +121,10 @@ export const mimoTtsManifest = {
},
{
name: "voiceClone", tag: "VoiceClone", mode: "structured",
- outputs: [mimoTtsEndpoints.voiceClone.requestType],
+ outputs: [
+ mimoTtsEndpoints.voiceClone.draftType,
+ ...Object.values(mimoTtsEndpoints.voiceClone.mediaBindings).map((binding) => binding.type),
+ ],
implementation: {
kind: "trusted-frontend-surface" as const,
locator: "@narratage/mimo-tts/voice-clone-surface",
diff --git a/packages/mimo-tts/src/surface.ts b/packages/mimo-tts/src/surface.ts
index 55ae5a97..a35e86e6 100644
--- a/packages/mimo-tts/src/surface.ts
+++ b/packages/mimo-tts/src/surface.ts
@@ -1,7 +1,13 @@
import { artifactTypes } from "@narratage/artifact";
-import type { GenerationPortValue } from "@narratage/generation";
+import {
+ generationPort,
+ sealGenerationMediaBinding,
+} from "@narratage/generation";
+import type { GenerationMediaPort, GenerationPortValue } from "@narratage/generation";
+import { exactModelMediaInputNames } from "@narratage/model-kit";
+import type { ExactModelMediaInput } from "@narratage/model-kit";
import { narrativeTypes } from "@narratage/narrative";
-import type { BlobRef, CanonicalValue } from "@narratage/protocol";
+import type { CanonicalValue } from "@narratage/protocol";
import type {
StructuredElement,
StructuredSurfaceHandler,
@@ -13,7 +19,7 @@ import { createMimoTtsAudioFragment } from "./fragment.js";
import {
mimoPresetVoices,
mimoTtsEndpoints,
- sealMimoTtsRequest,
+ sealMimoTtsRequestDraft,
} from "./index.js";
import type { MimoPresetVoice, MimoTtsModel } from "./index.js";
@@ -74,12 +80,15 @@ function speechText(reference: SurfaceResolvedReference, subject: string): strin
return value.speech;
}
-function sample(reference: SurfaceResolvedReference, subject: string): BlobRef {
- if (!sameType(reference.type, artifactTypes.blob) || reference.record?.value.kind !== "blob") {
- throw new Error(`${subject} must reference an authored audio BlobArtifact`);
+function sample(reference: SurfaceResolvedReference, subject: string): SurfaceResolvedReference {
+ if (!sameType(reference.type, artifactTypes.blob)) {
+ throw new Error(`${subject} must reference an audio BlobArtifact`);
+ }
+ const value = reference.record?.value;
+ if (value !== undefined && (value.kind !== "blob" || !value.mediaType.startsWith("audio/"))) {
+ throw new Error(`${subject} must be audio media`);
}
- if (!reference.record.value.mediaType.startsWith("audio/")) throw new Error(`${subject} must be audio media`);
- return reference.record.value;
+ return reference;
}
function body(element: StructuredElement, required: boolean): string | undefined {
@@ -103,22 +112,51 @@ function body(element: StructuredElement, required: boolean): string | undefined
function output(
element: StructuredElement,
endpoint: (typeof mimoTtsEndpoints)[keyof typeof mimoTtsEndpoints],
- request: ReturnType<typeof sealMimoTtsRequest>,
+ draft: ReturnType<typeof sealMimoTtsRequestDraft>,
+ media: readonly { readonly input: ExactModelMediaInput; readonly source: SurfaceResolvedReference }[] = [],
) {
const id = stringAttribute(element, "id");
- const requestId = `${id}.request`;
- const fragment = createMimoTtsAudioFragment(endpoint);
- return {
- records: [{
- id: requestId,
- type: endpoint.requestType,
- value: { kind: "inline" as const, value: request as unknown as CanonicalValue },
+ const draftId = `${id}.draft`;
+ const records: Array<{
+ readonly id: string;
+ readonly type: SurfaceResolvedReference["type"];
+ readonly value: { readonly kind: "inline"; readonly value: CanonicalValue };
+ readonly range: StructuredElement["range"];
+ }> = [{
+ id: draftId,
+ type: endpoint.draftType,
+ value: { kind: "inline", value: draft as unknown as CanonicalValue },
+ range: element.range,
+ }];
+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }> = {
+ draft: { kind: "record", id: draftId },
+ };
+ for (const item of media) {
+ const binding = endpoint.mediaBindings[item.input.port];
+ if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${item.input.port}`);
+ const port = generationPort(endpoint.ports, item.input.port);
+ if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${item.input.port} is not media`);
+ const names = exactModelMediaInputNames(item.input.name);
+ const bindingId = `${id}.${item.input.name}.binding`;
+ records.push({
+ id: bindingId,
+ type: binding.type,
+ value: {
+ kind: "inline",
+ value: sealGenerationMediaBinding(port as GenerationMediaPort, { role: "audio" }) as unknown as CanonicalValue,
+ },
range: element.range,
- }],
+ });
+ inputs[names.binding] = { kind: "record", id: bindingId };
+ inputs[names.artifact] = item.source.ref;
+ }
+ const fragment = createMimoTtsAudioFragment(endpoint, media.map((item) => item.input));
+ return {
+ records,
components: [{
id,
fragment: fragment.id,
- inputs: { request: { kind: "record" as const, id: requestId } },
+ inputs,
outputs: { audio: `${id}.audio` },
range: element.range,
}],
@@ -140,12 +178,12 @@ export const decodeMimoPresetSurface: StructuredSurfaceHandler = ({ element, res
const instruction = body(element, false);
const ports: Record<string, readonly GenerationPortValue[]> = { text: common(element, resolveReference), voice: [voice] };
if (instruction !== undefined) ports.instruction = [instruction];
- return output(element, mimoTtsEndpoints.preset, sealMimoTtsRequest("mimo-v2.5-tts", ports));
+ return output(element, mimoTtsEndpoints.preset, sealMimoTtsRequestDraft("mimo-v2.5-tts", ports));
};
export const decodeMimoVoiceDesignSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
attributes(element, ["id", "speech"]);
- return output(element, mimoTtsEndpoints.voiceDesign, sealMimoTtsRequest("mimo-v2.5-tts-voicedesign", {
+ return output(element, mimoTtsEndpoints.voiceDesign, sealMimoTtsRequestDraft("mimo-v2.5-tts-voicedesign", {
text: common(element, resolveReference),
voiceDescription: [body(element, true)!],
}));
@@ -154,10 +192,15 @@ export const decodeMimoVoiceDesignSurface: StructuredSurfaceHandler = ({ element
export const decodeMimoVoiceCloneSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
attributes(element, ["id", "speech", "sample"]);
const instruction = body(element, false);
+ const source = sample(resolved(element, "sample", resolveReference), `${element.name}.sample`);
const ports: Record<string, readonly GenerationPortValue[]> = {
text: common(element, resolveReference),
- sample: [{ role: "audio", artifact: sample(resolved(element, "sample", resolveReference), `${element.name}.sample`) }],
};
if (instruction !== undefined) ports.instruction = [instruction];
- return output(element, mimoTtsEndpoints.voiceClone, sealMimoTtsRequest("mimo-v2.5-tts-voiceclone", ports));
+ return output(
+ element,
+ mimoTtsEndpoints.voiceClone,
+ sealMimoTtsRequestDraft("mimo-v2.5-tts-voiceclone", ports),
+ [{ input: { name: "sample", port: "sample" }, source }],
+ );
};
diff --git a/packages/mimo-tts/test/mimo-tts.test.ts b/packages/mimo-tts/test/mimo-tts.test.ts
index bdf49947..2f98c039 100644
--- a/packages/mimo-tts/test/mimo-tts.test.ts
+++ b/packages/mimo-tts/test/mimo-tts.test.ts
@@ -95,13 +95,21 @@ test("the three author Surfaces are separate components with one ordinary audio
Calm and direct.
</mimo:VoiceClone>`)),
]);
- for (const result of results) {
+ for (const [index, result] of results.entries()) {
assert.equal(result.components.length, 1);
assert.deepEqual(Object.keys(result.components[0]!.outputs), ["audio"]);
- assert.deepEqual(result.fragments[0]!.operations.map((operation) => operation.producer.name), [
- result.fragments[0]!.operations[0]!.producer.name,
- generationProducers.primaryAudio.name,
- ]);
+ assert.deepEqual(result.fragments[0]!.operations.map((operation) => operation.producer.name), index === 2
+ ? [
+ "bind-request-mimo-v2.5-tts-voiceclone-sample",
+ "finalize-request-mimo-v2.5-tts-voiceclone",
+ "request-mimo-v2.5-tts-voiceclone",
+ generationProducers.primaryAudio.name,
+ ]
+ : [
+ index === 0 ? "finalize-request-mimo-v2.5-tts" : "finalize-request-mimo-v2.5-tts-voicedesign",
+ index === 0 ? "request-mimo-v2.5-tts" : "request-mimo-v2.5-tts-voicedesign",
+ generationProducers.primaryAudio.name,
+ ]);
assert.equal(result.records[0]!.value.kind, "inline");
const request = result.records[0]!.value.kind === "inline"
? result.records[0]!.value.value as Record<string, unknown> : {};
diff --git a/packages/model-kit/src/index.ts b/packages/model-kit/src/index.ts
index 0157ea87..1e642b6b 100644
--- a/packages/model-kit/src/index.ts
+++ b/packages/model-kit/src/index.ts
@@ -1,17 +1,31 @@
import type {
ComponentPackage,
ProducerFacet,
+ ProducerHandlerContext,
TypeValidatorFacet,
} from "@narratage/component-kit";
+import { artifactDependency, artifactTypes } from "@narratage/artifact";
import { sealGraphFragment } from "@narratage/elaborator";
import {
+ bindGenerationMedia,
+ finalizeGenerationRequestDraft,
generationManifestDigest,
generationModuleRef,
+ generationProducers,
generationTypes,
+ mediaBindingSchemaFromPort,
+ requestDraftSchemaFromPorts,
requestSchemaFromPorts,
+ verifyGenerationMediaBinding,
+ verifyRequestDraftAgainstPorts,
verifyRequestAgainstPorts,
} from "@narratage/generation";
-import type { GenerationPortTable } from "@narratage/generation";
+import type {
+ GenerationMediaBinding,
+ GenerationMediaPort,
+ GenerationPortTable,
+ GenerationRequestDraft,
+} from "@narratage/generation";
import {
canonicalize,
digestOf,
@@ -47,10 +61,29 @@ export type ExactModelEndpoint = {
readonly returns: TypeRef;
readonly implementationDigest: Digest;
readonly validatorDigest: Digest;
+ readonly draftType: TypeRef;
+ readonly draftValidatorDigest: Digest;
+ readonly finalizeProducer: ProducerRef;
+ readonly finalizeImplementationDigest: Digest;
+ readonly mediaBindings: Readonly<Record<string, ExactModelMediaBindingEndpoint>>;
readonly ports: GenerationPortTable;
readonly fragment: ReturnType<typeof sealGraphFragment>;
};
+export type ExactModelMediaBindingEndpoint = {
+ readonly port: string;
+ readonly type: TypeRef;
+ readonly producer: ProducerRef;
+ readonly implementationDigest: Digest;
+};
+
+export type ExactModelMediaInput = {
+ /** Stable local input name inside this Fragment shape. */
+ readonly name: string;
+ /** Exact model media port receiving the artifact. */
+ readonly port: string;
+};
+
export type ExactModelModule<Key extends string = string> = {
readonly module: ModuleRef;
readonly manifest: ModuleManifest;
@@ -74,6 +107,7 @@ function assert(condition: unknown, message: string): asserts condition {
function endpointRef(module: ModuleRef, spec: ExactModelEndpointSpec) {
const requestType = { module, name: spec.requestTypeName };
+ const draftType = { module, name: `${spec.requestTypeName}Draft` };
const capability = { module, name: spec.ports.model };
const producer = { module, name: spec.producerName };
const returns = spec.ports.result === "audio"
@@ -81,7 +115,34 @@ function endpointRef(module: ModuleRef, spec: ExactModelEndpointSpec) {
: spec.ports.result === "image" ? generationTypes.imageSet : generationTypes.videoSet;
const implementationDigest = digestOf(`${module.name}/${spec.producerName}@1`);
const validatorDigest = digestOf(`${module.name}/validate-${spec.requestTypeName}@1`);
- return { requestType, capability, producer, returns, implementationDigest, validatorDigest };
+ const draftValidatorDigest = digestOf(`${module.name}/validate-${spec.requestTypeName}-draft@1`);
+ const finalizeProducer = { module, name: `finalize-${spec.producerName}` };
+ const finalizeImplementationDigest = digestOf(`${module.name}/${finalizeProducer.name}@1`);
+ const mediaBindings = Object.fromEntries(spec.ports.ports
+ .filter((port): port is GenerationMediaPort => port.value.kind === "media")
+ .map((port) => [port.name, {
+ port: port.name,
+ type: { module, name: `${spec.requestTypeName}${pascal(port.name)}Binding` },
+ producer: { module, name: `bind-${spec.producerName}-${port.name}` },
+ implementationDigest: digestOf(`${module.name}/bind-${spec.producerName}-${port.name}@1`),
+ } satisfies ExactModelMediaBindingEndpoint]));
+ return {
+ requestType,
+ draftType,
+ capability,
+ producer,
+ returns,
+ implementationDigest,
+ validatorDigest,
+ draftValidatorDigest,
+ finalizeProducer,
+ finalizeImplementationDigest,
+ mediaBindings,
+ };
+}
+
+function pascal(value: string): string {
+ return value.replace(/(^|[^a-zA-Z0-9]+)([a-zA-Z0-9])/gu, (_whole, _boundary, letter: string) => letter.toUpperCase());
}
function inlineRequest(
@@ -92,6 +153,14 @@ function inlineRequest(
return canonicalize(value.value);
}
+function inlineValue<T>(
+ value: { readonly kind: string; readonly value?: CanonicalValue },
+ subject: string,
+): T {
+ assert(value.kind === "inline" && value.value !== undefined, `${subject} must be inline`);
+ return canonicalize(value.value) as unknown as T;
+}
+
/**
* Builds the repetitive nominal shell around an exact model request. The model package still owns
* every field, constraint and model name; this helper only wires Type -> Producer -> Need -> Fragment.
@@ -112,39 +181,90 @@ export function defineExactModelModule<const Key extends string>(
format: "svml.module@1",
name: options.module.name,
version: options.module.version,
- dependencies: [{ module: generationModuleRef, digest: generationManifestDigest }],
- types: endpointData.map((item) => ({
- name: item.requestType.name,
- schema: requestSchemaFromPorts(item.spec.ports),
- validator: {
- abi: "svml.type-validator@1",
- implementation: {
- kind: "registered",
- locator: `${options.module.name}/validate-${item.spec.key}`,
- digest: item.validatorDigest,
+ dependencies: [
+ { module: generationModuleRef, digest: generationManifestDigest },
+ artifactDependency,
+ ],
+ types: endpointData.flatMap((item) => [
+ {
+ name: item.requestType.name,
+ schema: requestSchemaFromPorts(item.spec.ports),
+ validator: {
+ abi: "svml.type-validator@1" as const,
+ implementation: {
+ kind: "registered" as const,
+ locator: `${options.module.name}/validate-${item.spec.key}`,
+ digest: item.validatorDigest,
+ },
},
},
- })),
+ {
+ name: item.draftType.name,
+ schema: requestDraftSchemaFromPorts(item.spec.ports),
+ validator: {
+ abi: "svml.type-validator@1" as const,
+ implementation: {
+ kind: "registered" as const,
+ locator: `${options.module.name}/validate-${item.spec.key}-draft`,
+ digest: item.draftValidatorDigest,
+ },
+ },
+ },
+ ...Object.values(item.mediaBindings).map((binding) => {
+ const port = item.spec.ports.ports.find((candidate): candidate is GenerationMediaPort =>
+ candidate.name === binding.port && candidate.value.kind === "media");
+ assert(port !== undefined, `${item.spec.key} media binding ${binding.port} has no port`);
+ return { name: binding.type.name, schema: mediaBindingSchemaFromPort(port) };
+ }),
+ ]),
capabilities: endpointData.map((item) => ({
name: item.capability.name,
returns: item.returns,
})),
surfaces: [],
- producers: endpointData.map((item) => ({
- name: item.producer.name,
- inputs: [{ name: "request", type: item.requestType }],
- outputs: [],
- needs: [{
- name: "generation",
- capability: item.capability,
- returns: item.returns,
- }],
- implementation: {
- kind: "registered",
- locator: `${options.module.name}/${item.spec.key}`,
- digest: item.implementationDigest,
+ producers: endpointData.flatMap((item) => [
+ {
+ name: item.producer.name,
+ inputs: [{ name: "request", type: item.requestType }],
+ outputs: [],
+ needs: [{
+ name: "generation",
+ capability: item.capability,
+ returns: item.returns,
+ }],
+ implementation: {
+ kind: "registered" as const,
+ locator: `${options.module.name}/${item.spec.key}`,
+ digest: item.implementationDigest,
+ },
},
- })),
+ ...Object.values(item.mediaBindings).map((binding) => ({
+ name: binding.producer.name,
+ inputs: [
+ { name: "draft", type: item.draftType },
+ { name: "binding", type: binding.type },
+ { name: "artifact", type: artifactTypes.blob },
+ ],
+ outputs: [{ name: "draft", type: item.draftType }],
+ needs: [],
+ implementation: {
+ kind: "registered" as const,
+ locator: `${options.module.name}/${item.spec.key}/bind-${binding.port}`,
+ digest: binding.implementationDigest,
+ },
+ })),
+ {
+ name: item.finalizeProducer.name,
+ inputs: [{ name: "draft", type: item.draftType }],
+ outputs: [{ name: "request", type: item.requestType }],
+ needs: [],
+ implementation: {
+ kind: "registered" as const,
+ locator: `${options.module.name}/${item.spec.key}/finalize`,
+ digest: item.finalizeImplementationDigest,
+ },
+ },
+ ]),
};
const endpoints = Object.fromEntries(endpointData.map((item): [Key, ExactModelEndpoint] => {
@@ -173,6 +293,11 @@ export function defineExactModelModule<const Key extends string>(
returns: item.returns,
implementationDigest: item.implementationDigest,
validatorDigest: item.validatorDigest,
+ draftType: item.draftType,
+ draftValidatorDigest: item.draftValidatorDigest,
+ finalizeProducer: item.finalizeProducer,
+ finalizeImplementationDigest: item.finalizeImplementationDigest,
+ mediaBindings: item.mediaBindings,
ports: item.spec.ports,
fragment,
} satisfies ExactModelEndpoint];
@@ -185,24 +310,160 @@ export function defineExactModelModule<const Key extends string>(
endpoints: endpoints as Readonly<Record<Key, ExactModelEndpoint>>,
component: {
name: options.module.name,
- validators: endpointData.map((item) => ({
+ validators: endpointData.flatMap((item) => [{
type: item.requestType,
implementationDigest: item.validatorDigest,
handler({ value }) {
verifyRequestAgainstPorts(item.spec.ports, inlineRequest(value, item.spec.key));
},
- })),
- producers: endpointData.map((item) => ({
- producer: item.producer,
- implementationDigest: item.implementationDigest,
- handler: ({ inputs }) => {
- const requestRecord = inputs.request;
- assert(requestRecord !== undefined, `${item.spec.key} request input is missing`);
- const request = inlineRequest(requestRecord.value, item.spec.key);
- verifyRequestAgainstPorts(item.spec.ports, request);
- return { outputs: {}, needs: { generation: request } };
+ }, {
+ type: item.draftType,
+ implementationDigest: item.draftValidatorDigest,
+ handler({ value }) {
+ verifyRequestDraftAgainstPorts(item.spec.ports, inlineRequest(value, `${item.spec.key} draft`));
},
- })),
+ }]),
+ producers: endpointData.flatMap((item) => [
+ {
+ producer: item.producer,
+ implementationDigest: item.implementationDigest,
+ handler: ({ inputs }: ProducerHandlerContext) => {
+ const requestRecord = inputs.request;
+ assert(requestRecord !== undefined, `${item.spec.key} request input is missing`);
+ const request = inlineRequest(requestRecord.value, item.spec.key);
+ verifyRequestAgainstPorts(item.spec.ports, request);
+ return { outputs: {}, needs: { generation: request } };
+ },
+ },
+ ...Object.values(item.mediaBindings).map((binding) => ({
+ producer: binding.producer,
+ implementationDigest: binding.implementationDigest,
+ handler: ({ inputs }: ProducerHandlerContext) => {
+ const draft = inlineValue<GenerationRequestDraft>(inputs.draft!.value, `${item.spec.key} draft`);
+ const value = inlineValue<GenerationMediaBinding>(inputs.binding!.value, `${binding.port} binding`);
+ const port = item.spec.ports.ports.find((candidate): candidate is GenerationMediaPort =>
+ candidate.name === binding.port && candidate.value.kind === "media");
+ assert(port !== undefined, `${item.spec.key} media binding ${binding.port} has no port`);
+ verifyGenerationMediaBinding(port, value);
+ const artifact = inputs.artifact!.value;
+ assert(artifact.kind === "blob", `${binding.port} artifact must be a Blob`);
+ return {
+ outputs: {
+ draft: {
+ kind: "inline" as const,
+ value: canonicalize(bindGenerationMedia(item.spec.ports, draft, binding.port, value, artifact)),
+ },
+ },
+ needs: {},
+ };
+ },
+ })),
+ {
+ producer: item.finalizeProducer,
+ implementationDigest: item.finalizeImplementationDigest,
+ handler: ({ inputs }) => ({
+ outputs: {
+ request: {
+ kind: "inline" as const,
+ value: canonicalize(finalizeGenerationRequestDraft(
+ item.spec.ports,
+ inlineValue<GenerationRequestDraft>(inputs.draft!.value, `${item.spec.key} draft`),
+ )),
+ },
+ },
+ needs: {},
+ }),
+ },
+ ]),
},
};
}
+
+export function exactModelMediaInputNames(name: string): {
+ readonly artifact: string;
+ readonly binding: string;
+} {
+ assert(name.trim().length > 0, "Exact model media input name is empty");
+ return { artifact: `${name}:artifact`, binding: `${name}:binding` };
+}
+
+/**
+ * Expand one exact-model invocation into a deterministic request-assembly
+ * graph. Every runtime-produced media artifact remains a real Fragment input;
+ * the fold only constructs the Provider-facing request after those inputs are
+ * available.
+ */
+export function createExactModelPrimaryGenerationFragment(
+ endpoint: ExactModelEndpoint,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+) {
+ const names = mediaInputs.map((item) => item.name);
+ assert(new Set(names).size === names.length, "Exact model media input names must be unique");
+ const inputs = [{ name: "draft", type: endpoint.draftType }];
+ const operations: Array<{
+ readonly id: string;
+ readonly producer: ProducerRef;
+ readonly inputs: Readonly<Record<string, { readonly kind: "fragment-input"; readonly name: string } | { readonly kind: "fragment-operation"; readonly operation: string }>>;
+ readonly result: { readonly kind: "output"; readonly name: string } | { readonly kind: "need"; readonly name: string; readonly accepts: "exact" };
+ }> = [];
+ const input = (name: string) => ({ kind: "fragment-input" as const, name });
+ const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+ let draft = input("draft") as ReturnType<typeof input> | ReturnType<typeof operation>;
+
+ for (const [index, item] of mediaInputs.entries()) {
+ const binding = endpoint.mediaBindings[item.port];
+ assert(binding !== undefined, `${endpoint.ports.model} has no media port ${item.port}`);
+ const inputNames = exactModelMediaInputNames(item.name);
+ inputs.push({ name: inputNames.binding, type: binding.type });
+ inputs.push({ name: inputNames.artifact, type: artifactTypes.blob });
+ const id = `bind:${String(index + 1).padStart(4, "0")}:${item.port}`;
+ operations.push({
+ id,
+ producer: binding.producer,
+ inputs: {
+ draft,
+ binding: input(inputNames.binding),
+ artifact: input(inputNames.artifact),
+ },
+ result: { kind: "output", name: "draft" },
+ });
+ draft = operation(id);
+ }
+
+ operations.push({
+ id: "finalize-request",
+ producer: endpoint.finalizeProducer,
+ inputs: { draft },
+ result: { kind: "output", name: "request" },
+ });
+ operations.push({
+ id: "generate",
+ producer: endpoint.producer,
+ inputs: { request: operation("finalize-request") },
+ result: { kind: "need", name: "generation", accepts: "exact" },
+ });
+ const result = endpoint.ports.result;
+ const primaryProducer = result === "audio"
+ ? generationProducers.primaryAudio
+ : result === "image" ? generationProducers.primaryImage : generationProducers.primaryVideo;
+ operations.push({
+ id: `select-primary-${result}`,
+ producer: primaryProducer,
+ inputs: { set: operation("generate") },
+ result: { kind: "output", name: result },
+ });
+ const semanticInputs = inputs.map((entry) => entry.name);
+ const shape = mediaInputs.map((item) => `${item.name}=${item.port}`).join(",") || "no-media";
+ return sealGraphFragment({
+ name: `${endpoint.producer.module.name}/${endpoint.key}-assembled-primary-${result}[${shape}]@1`,
+ inputs,
+ operations,
+ exports: [{
+ name: result,
+ type: artifactTypes.blob,
+ root: operation(`select-primary-${result}`),
+ semanticInputs,
+ fidelity: "exact",
+ }],
+ });
+}
diff --git a/packages/model-kit/test/model-kit.test.ts b/packages/model-kit/test/model-kit.test.ts
new file mode 100644
index 00000000..84c88661
--- /dev/null
+++ b/packages/model-kit/test/model-kit.test.ts
@@ -0,0 +1,86 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ sealGenerationMediaBinding,
+ sealGenerationRequestDraft,
+ sealGenerationPortTable,
+} from "@narratage/generation";
+import {
+ createExactModelPrimaryGenerationFragment,
+ defineExactModelModule,
+} from "@narratage/model-kit";
+import { digestOf } from "@narratage/protocol";
+
+const ports = sealGenerationPortTable({
+ contract: "svml.generation-ports@1",
+ model: "graph-native-image",
+ result: "image",
+ ports: [
+ { name: "prompt", value: { kind: "text" }, minItems: 1, maxItems: 1 },
+ { name: "images", value: { kind: "media", accepts: ["image"] }, minItems: 1, maxItems: 3 },
+ ],
+ requires: [],
+});
+
+const definition = defineExactModelModule({
+ module: { name: "@test/graph-native-image", version: "1" },
+ endpoints: [{
+ key: "image",
+ requestTypeName: "GraphNativeImageRequest",
+ producerName: "request-graph-native-image",
+ ports,
+ }],
+});
+
+test("one exact model definition owns draft, media binding, finalization and generation contracts", async () => {
+ const endpoint = definition.endpoints.image!;
+ assert.equal(endpoint.draftType.name, "GraphNativeImageRequestDraft");
+ assert.equal(endpoint.mediaBindings.images?.type.name, "GraphNativeImageRequestImagesBinding");
+ assert.ok(definition.manifest.producers.some((producer) => producer.name === endpoint.finalizeProducer.name));
+
+ const artifact = {
+ kind: "blob" as const,
+ digest: digestOf("graph-native-image"),
+ size: 4,
+ mediaType: "image/png",
+ };
+ const draft = sealGenerationRequestDraft(ports, { prompt: ["draw it"] });
+ const mediaPort = ports.ports.find((port) => port.name === "images");
+ assert.ok(mediaPort?.value.kind === "media");
+ const binding = sealGenerationMediaBinding(mediaPort as never, { role: "image" });
+ const bindFacet = definition.component.producers.find((facet) => facet.producer.name === endpoint.mediaBindings.images?.producer.name);
+ assert.ok(bindFacet);
+ const bound = await bindFacet.handler({ inputs: {
+ draft: { value: { kind: "inline", value: draft } },
+ binding: { value: { kind: "inline", value: binding } },
+ artifact: { value: artifact },
+ } } as never);
+ const finalizeFacet = definition.component.producers.find((facet) => facet.producer.name === endpoint.finalizeProducer.name);
+ assert.ok(finalizeFacet);
+ const finalized = await finalizeFacet.handler({ inputs: { draft: { value: bound.outputs.draft! } } } as never);
+ assert.equal(finalized.outputs.request?.kind, "inline");
+ if (finalized.outputs.request?.kind !== "inline") return;
+ const request = finalized.outputs.request.value as Record<string, unknown>;
+ assert.equal(request.contract, "svml.generation-request@1");
+});
+
+test("the dynamic Fragment exposes every media artifact as an explicit semantic input", () => {
+ const fragment = createExactModelPrimaryGenerationFragment(definition.endpoints.image!, [
+ { name: "first", port: "images" },
+ { name: "second", port: "images" },
+ ]);
+ assert.deepEqual(fragment.inputs.map((input) => input.name), [
+ "draft", "first:artifact", "first:binding", "second:artifact", "second:binding",
+ ]);
+ assert.deepEqual(fragment.operations.map((operation) => operation.producer.name), [
+ "bind-request-graph-native-image-images",
+ "bind-request-graph-native-image-images",
+ "finalize-request-graph-native-image",
+ "request-graph-native-image",
+ "select-primary-image",
+ ]);
+ assert.deepEqual(fragment.exports[0]?.semanticInputs, [
+ "draft", "first:artifact", "first:binding", "second:artifact", "second:binding",
+ ]);
+});
diff --git a/packages/seedance-speaker/src/fragment.ts b/packages/seedance-speaker/src/fragment.ts
index 486a6436..972a79a9 100644
--- a/packages/seedance-speaker/src/fragment.ts
+++ b/packages/seedance-speaker/src/fragment.ts
@@ -1,52 +1,16 @@
-import { speechTypes } from "@narratage/speech";
-import { artifactTypes } from "@narratage/artifact";
-import { sealGraphFragment } from "@narratage/elaborator";
-import { generationProducers } from "@narratage/generation";
-import type { ExactModelEndpoint } from "@narratage/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput } from "@narratage/model-kit";
import type { ProducerRef } from "@narratage/protocol";
-import { seedanceTypes } from "@narratage/seedance";
-
-const input = (name: string) => ({ kind: "fragment-input" as const, name });
-const operation = (id: string) => ({ kind: "fragment-operation" as const, operation: id });
+import { createSeedanceSpeechGenerationFragment } from "@narratage/seedance";
+/**
+ * Speaker is an authoring Kit, not another execution protocol. Once its prompt
+ * program is compiled, it uses Seedance's ordinary graph-native request
+ * assembly, including every runtime-produced media edge.
+ */
export function createSeedanceSpeakerTakeFragment(
endpoint: ExactModelEndpoint,
compileRequestProducer: ProducerRef,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
) {
- return sealGraphFragment({
- name: `@narratage/seedance-speaker/${endpoint.key}-take@1`,
- inputs: [
- { name: "program", type: seedanceTypes.speechSpine },
- { name: "duration", type: speechTypes.duration },
- ],
- operations: [
- {
- id: "compile-request",
- producer: compileRequestProducer,
- inputs: { program: input("program"), duration: input("duration") },
- result: { kind: "output", name: "request" },
- },
- {
- id: "generate",
- producer: endpoint.producer,
- inputs: { request: operation("compile-request") },
- result: { kind: "need", name: "generation", accepts: "exact" },
- },
- {
- id: "select-primary-video",
- producer: generationProducers.primaryVideo,
- inputs: { set: operation("generate") },
- result: { kind: "output", name: "video" },
- },
- ],
- exports: [
- {
- name: "video",
- type: artifactTypes.blob,
- root: operation("select-primary-video"),
- semanticInputs: ["program", "duration"],
- fidelity: "exact",
- },
- ],
- });
+ return createSeedanceSpeechGenerationFragment(endpoint, compileRequestProducer, mediaInputs);
}
diff --git a/packages/seedance-speaker/src/kit.ts b/packages/seedance-speaker/src/kit.ts
index 38f65672..8b8ae7d8 100644
--- a/packages/seedance-speaker/src/kit.ts
+++ b/packages/seedance-speaker/src/kit.ts
@@ -1,4 +1,3 @@
-import { assertGenerationBlobRef } from "@narratage/generation";
import {
sealPromptKitInvocation,
verifyPromptKitInvocation,
@@ -11,8 +10,6 @@ import {
} from "@narratage/seedance";
import type { SeedanceSpeechProgram } from "@narratage/seedance";
-import type { BlobRef } from "@narratage/protocol";
-
import type { SpeakerTakeIntent } from "./types.js";
export const speakerMethodDefaults = {
@@ -72,7 +69,6 @@ export function verifySpeakerTakeIntent(value: unknown): asserts value is Speake
if ((item.kind !== "image" && item.kind !== "audio") || typeof item.role !== "string" || item.role.length === 0) {
throw new Error("Speaker reference is invalid");
}
- assertGenerationBlobRef(item.artifact, `${item.kind}/` as "image/" | "audio/");
}
}
@@ -104,18 +100,6 @@ export function bindSpeakerPromptKit(intent: SpeakerTakeIntent): PromptKitInvoca
return invocation;
}
-/** The model keeps one port per reference modality; the Speaker groups its takes accordingly. */
-function referencePortsOf(
- references: SpeakerTakeIntent["references"],
-): Record<string, readonly { readonly role: "image" | "video" | "audio"; readonly artifact: BlobRef }[]> {
- const ports: Record<string, { readonly role: "image" | "video" | "audio"; readonly artifact: BlobRef }[]> = {};
- const names = { image: "referenceImage", video: "referenceVideo", audio: "referenceAudio" } as const;
- for (const item of references) {
- (ports[names[item.kind]] ??= []).push({ role: item.kind, artifact: item.artifact });
- }
- return ports;
-}
-
export function renderSpeakerSpeechProgram(
program: PromptProgram,
intent: SpeakerTakeIntent,
@@ -127,7 +111,6 @@ export function renderSpeakerSpeechProgram(
model: intent.model,
ports: {
prompt: [program.blocks.map((item) => item.text).join(program.separator)],
- ...referencePortsOf(intent.references),
resolution: [intent.resolution],
aspectRatio: [intent.aspectRatio],
generateAudio: [true],
diff --git a/packages/seedance-speaker/src/manifest.ts b/packages/seedance-speaker/src/manifest.ts
index 5c78d904..e772c620 100644
--- a/packages/seedance-speaker/src/manifest.ts
+++ b/packages/seedance-speaker/src/manifest.ts
@@ -12,6 +12,7 @@ import type { ModuleManifest } from "@narratage/protocol";
import {
seedanceManifestDigest,
seedanceModuleRef,
+ seedanceEndpointsByModel,
seedanceTypes,
} from "@narratage/seedance";
import { svsManifest, svsRecipeType } from "@narratage/svs";
@@ -40,7 +41,13 @@ export const seedanceSpeakerManifest: ModuleManifest = {
name: "take",
tag: "Take",
mode: "structured",
- outputs: [promptKitTypes.program, seedanceTypes.speechSpine, artifactTypes.blob],
+ outputs: [
+ promptKitTypes.program,
+ seedanceTypes.speechSpine,
+ artifactTypes.blob,
+ ...Object.values(seedanceEndpointsByModel).flatMap((endpoint) =>
+ Object.values(endpoint.mediaBindings).map((binding) => binding.type)),
+ ],
implementation: {
kind: "trusted-frontend-surface",
locator: "@narratage/seedance-speaker/take-surface",
diff --git a/packages/seedance-speaker/src/surface.ts b/packages/seedance-speaker/src/surface.ts
index 1443e034..3e53b3fe 100644
--- a/packages/seedance-speaker/src/surface.ts
+++ b/packages/seedance-speaker/src/surface.ts
@@ -4,6 +4,10 @@ import { speechTypes } from "@narratage/speech";
import type { SpeechDuration } from "@narratage/speech";
import { artifactTypes } from "@narratage/artifact";
import type { CanonicalValue, StoredValue } from "@narratage/protocol";
+import { generationPort, sealGenerationMediaBinding } from "@narratage/generation";
+import type { GenerationMediaPort } from "@narratage/generation";
+import { exactModelMediaInputNames } from "@narratage/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput } from "@narratage/model-kit";
import {
compilePromptKit,
promptKitTypes,
@@ -88,11 +92,15 @@ function inline<T>(reference: SurfaceResolvedReference, subject: string): T {
return value.value as unknown as T;
}
-function blob(reference: SurfaceResolvedReference, subject: string) {
- if (!sameType(reference.type, artifactTypes.blob) || reference.record?.value.kind !== "blob") {
- throw new Error(`${subject} must reference an authored BlobArtifact`);
+function mediaReference(reference: SurfaceResolvedReference, kind: "image" | "audio", subject: string) {
+ if (!sameType(reference.type, artifactTypes.blob)) {
+ throw new Error(`${subject} must reference a BlobArtifact`);
}
- return reference.record.value;
+ const value = reference.record?.value;
+ if (value !== undefined && (value.kind !== "blob" || !value.mediaType.startsWith(`${kind}/`))) {
+ throw new Error(`${subject} is not ${kind} media`);
+ }
+ return reference;
}
function recipeValue(
@@ -149,11 +157,13 @@ function dialogueValue(
return value;
}
+type ResolvedSpeakerReference = SpeakerReference & { readonly source: SurfaceResolvedReference };
+
function references(
element: StructuredElement,
resolveReference: (path: string) => SurfaceResolvedReference | undefined,
-): SpeakerReference[] {
- const result: SpeakerReference[] = [];
+): ResolvedSpeakerReference[] {
+ const result: ResolvedSpeakerReference[] = [];
for (const child of element.children) {
if (child.kind === "text") {
if (child.value.trim().length > 0) throw new Error(`${element.name} accepts only Reference children`);
@@ -164,19 +174,57 @@ function references(
const kinds = (["image", "audio"] as const).filter((kind) => child.attributes[kind] !== undefined);
if (kinds.length !== 1) throw new Error(`${child.name} requires exactly one of image or audio`);
const kind = kinds[0]!;
- const artifact = blob(resolved(child, kind, resolveReference), `${child.name}.${kind}`);
- if (!artifact.mediaType.startsWith(`${kind}/`)) throw new Error(`${child.name}.${kind} is not ${kind} media`);
+ const source = mediaReference(resolved(child, kind, resolveReference), kind, `${child.name}.${kind}`);
const rawRole = child.attributes.role;
const role = rawRole === undefined
? kind === "image" ? "character-and-scene" : "voice-timbre"
: typeof rawRole === "string" && rawRole.trim().length > 0
? rawRole.trim()
: (() => { throw new Error(`${child.name}.role must be a non-empty string`); })();
- result.push({ kind, artifact, role });
+ result.push({ kind, source, role });
}
return result;
}
+function assembledReferences(
+ id: string,
+ endpoint: ExactModelEndpoint,
+ values: readonly ResolvedSpeakerReference[],
+ range: StructuredElement["range"],
+) {
+ const mediaInputs: ExactModelMediaInput[] = [];
+ const records: Array<{
+ readonly id: string;
+ readonly type: SurfaceResolvedReference["type"];
+ readonly value: { readonly kind: "inline"; readonly value: CanonicalValue };
+ readonly range: StructuredElement["range"];
+ }> = [];
+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }> = {};
+ values.forEach((value, index) => {
+ const name = `media-${String(index + 1).padStart(4, "0")}`;
+ const portName = value.kind === "image" ? "referenceImage" : "referenceAudio";
+ const endpointBinding = endpoint.mediaBindings[portName];
+ if (endpointBinding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${portName}`);
+ const port = generationPort(endpoint.ports, portName);
+ if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${portName} is not media`);
+ const bindingId = `${id}.${name}.binding`;
+ records.push({
+ id: bindingId,
+ type: endpointBinding.type,
+ value: {
+ kind: "inline",
+ value: sealGenerationMediaBinding(port as GenerationMediaPort, { role: value.kind }) as unknown as CanonicalValue,
+ },
+ range,
+ });
+ const names = exactModelMediaInputNames(name);
+ inputs[names.binding] = { kind: "record", id: bindingId };
+ inputs[names.artifact] = value.source.ref;
+ mediaInputs.push({ name, port: portName });
+ });
+ return { mediaInputs, records, inputs };
+}
+
function recipeString(
recipe: SvsRecipe,
name: string,
@@ -247,7 +295,7 @@ export const decodeSeedanceSpeakerTakeSurface: StructuredSurfaceHandler = ({ ele
segment: {
dialogue: dialogue.dialogue,
},
- references: refs,
+ references: refs.map(({ source: _source, ...reference }) => reference),
...(actionPrompt === undefined ? {} : { actionPrompt }),
...(extraPrompt === undefined ? {} : { extraPrompt }),
});
@@ -260,9 +308,12 @@ export const decodeSeedanceSpeakerTakeSurface: StructuredSurfaceHandler = ({ ele
}
const promptId = `${id}.prompt`;
const programId = `${id}.program`;
+ const endpoint = seedanceEndpointsByModel[selectedModel];
+ const assembled = assembledReferences(id, endpoint, refs, element.range);
const fragment = createSeedanceSpeakerTakeFragment(
- seedanceEndpointsByModel[selectedModel],
+ endpoint,
seedanceSpeechCompileProducers[selectedModel],
+ assembled.mediaInputs,
);
return {
records: [{
@@ -275,13 +326,14 @@ export const decodeSeedanceSpeakerTakeSurface: StructuredSurfaceHandler = ({ ele
type: seedanceTypes.speechSpine,
value: { kind: "inline", value: program as unknown as CanonicalValue },
range: element.range,
- }],
+ }, ...assembled.records],
components: [{
id,
fragment: fragment.id,
inputs: {
program: { kind: "record", id: programId },
duration: duration.ref,
+ ...assembled.inputs,
},
outputs: {
video: `${id}.video`,
diff --git a/packages/seedance-speaker/src/types.ts b/packages/seedance-speaker/src/types.ts
index c2409c75..245af86d 100644
--- a/packages/seedance-speaker/src/types.ts
+++ b/packages/seedance-speaker/src/types.ts
@@ -1,9 +1,8 @@
-import type { CanonicalValue, BlobRef } from "@narratage/protocol";
+import type { CanonicalValue } from "@narratage/protocol";
import type { SeedanceModel } from "@narratage/seedance";
export type SpeakerReference = {
readonly kind: "image" | "audio";
- readonly artifact: BlobRef;
readonly role: string;
};
diff --git a/packages/seedance-speaker/test/kit.test.ts b/packages/seedance-speaker/test/kit.test.ts
index bdd57e2c..863010be 100644
--- a/packages/seedance-speaker/test/kit.test.ts
+++ b/packages/seedance-speaker/test/kit.test.ts
@@ -7,7 +7,6 @@ import {
promptKitSpecFromSvsRecipes,
verifyPromptProgram,
} from "@narratage/prompt-kit";
-import { digestOf } from "@narratage/protocol";
import { parseSvs } from "@narratage/svs";
import {
bindSpeakerPromptKit,
@@ -26,20 +25,6 @@ const officialUgcV1KitSpec = promptKitSpecFromSvsRecipes(
"official-ugc-v1",
);
-const image = {
- kind: "blob" as const,
- digest: digestOf("speaker-image"),
- size: 100,
- mediaType: "image/png",
-};
-
-const audio = {
- kind: "blob" as const,
- digest: digestOf("speaker-audio"),
- size: 200,
- mediaType: "audio/mpeg",
-};
-
function intent() {
return sealSpeakerTakeIntent({
contract: "svml.seedance-speaker-take-intent@1",
@@ -54,8 +39,8 @@ function intent() {
dialogue: "HOST: Say exactly these words.",
},
references: [
- { kind: "image", artifact: image, role: "character-and-scene" },
- { kind: "audio", artifact: audio, role: "voice-timbre" },
+ { kind: "image", role: "character-and-scene" },
+ { kind: "audio", role: "voice-timbre" },
],
});
}
@@ -87,21 +72,15 @@ test("official UGC Kit emits an ordered generic Prompt Program", () => {
assert.match(prompt.blocks[3]?.text ?? "", /@audio1/u);
});
-test("rendering the generic Prompt Program preserves exact Seedance method and references", () => {
+test("rendering the generic Prompt Program preserves exact Seedance method while media stays on graph edges", () => {
const take = intent();
const prompt = compilePromptKit(officialUgcV1KitSpec, bindSpeakerPromptKit(take));
const program = renderSpeakerSpeechProgram(prompt, take);
assert.equal(program.model, "seedance-2-mini");
assert.deepEqual(program.ports.resolution, ["720p"]);
assert.deepEqual(program.ports.aspectRatio, ["9:16"]);
- assert.deepEqual(
- (program.ports.referenceImage ?? []).map((item) => (item as { readonly role: string }).role),
- ["image"],
- );
- assert.deepEqual(
- (program.ports.referenceAudio ?? []).map((item) => (item as { readonly role: string }).role),
- ["audio"],
- );
+ assert.equal(program.ports.referenceImage, undefined);
+ assert.equal(program.ports.referenceAudio, undefined);
assert.deepEqual(program.ports.prompt, [prompt.blocks.map((item) => item.text).join("\n\n")]);
assert.equal(program.ports.duration, undefined, "duration arrives from speech estimation");
});
@@ -110,7 +89,7 @@ test("Speaker rejects invalid media while Prompt Kit rejects unknown parameters"
const base = intent();
assert.throws(() => sealSpeakerTakeIntent({
...base,
- references: [{ kind: "audio", artifact: audio, role: "voice-timbre" }],
+ references: [{ kind: "audio", role: "voice-timbre" }],
}), /requires 1-9 image references/u);
assert.throws(() => sealSpeakerTakeIntent({ ...base, resolution: "1080p" }), /at most 720p/u);
const unknown = sealSpeakerTakeIntent({ ...base, promptParameters: { "imaginary-axis": "value" } });
diff --git a/packages/seedance/src/fragment.ts b/packages/seedance/src/fragment.ts
index a19656f7..cdc3d83a 100644
--- a/packages/seedance/src/fragment.ts
+++ b/packages/seedance/src/fragment.ts
@@ -2,7 +2,11 @@ import { speechTypes } from "@narratage/speech";
import { artifactTypes } from "@narratage/artifact";
import { sealGraphFragment } from "@narratage/elaborator";
import { generationProducers } from "@narratage/generation";
-import type { ExactModelEndpoint } from "@narratage/model-kit";
+import {
+ createExactModelPrimaryGenerationFragment,
+ exactModelMediaInputNames,
+} from "@narratage/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput } from "@narratage/model-kit";
import type { ProducerRef } from "@narratage/protocol";
const input = (name: string) => ({ kind: "fragment-input" as const, name });
@@ -40,41 +44,74 @@ export function createSeedanceGenerationFragment(endpoint: ExactModelEndpoint) {
});
}
+/** Request assembly used by author Surfaces whose media references may be runtime outputs. */
+export function createSeedanceAssembledGenerationFragment(
+ endpoint: ExactModelEndpoint,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+) {
+ return createExactModelPrimaryGenerationFragment(endpoint, mediaInputs);
+}
+
export function createSeedanceSpeechGenerationFragment(
endpoint: ExactModelEndpoint,
compileProducer: ProducerRef,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
) {
+ const inputs = [
+ { name: "program", type: { module: endpoint.producer.module, name: "SpeechProgram" } },
+ { name: "duration", type: speechTypes.duration },
+ ];
+ const operations: Array<import("@narratage/elaborator").FragmentOperation> = [{
+ id: "compile-draft",
+ producer: compileProducer,
+ inputs: { program: input("program"), duration: input("duration") },
+ result: { kind: "output", name: "draft" },
+ }];
+ let draft = operation("compile-draft") as ReturnType<typeof input> | ReturnType<typeof operation>;
+ for (const [index, item] of mediaInputs.entries()) {
+ const binding = endpoint.mediaBindings[item.port];
+ if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${item.port}`);
+ const names = exactModelMediaInputNames(item.name);
+ inputs.push({ name: names.binding, type: binding.type }, { name: names.artifact, type: artifactTypes.blob });
+ const id = `bind:${String(index + 1).padStart(4, "0")}:${item.port}`;
+ operations.push({
+ id,
+ producer: binding.producer,
+ inputs: { draft, binding: input(names.binding), artifact: input(names.artifact) },
+ result: { kind: "output", name: "draft" },
+ });
+ draft = operation(id);
+ }
+ operations.push(
+ {
+ id: "finalize-request",
+ producer: endpoint.finalizeProducer,
+ inputs: { draft },
+ result: { kind: "output", name: "request" },
+ },
+ {
+ id: "generate",
+ producer: endpoint.producer,
+ inputs: { request: operation("finalize-request") },
+ result: { kind: "need", name: "generation", accepts: "exact" },
+ },
+ {
+ id: "select-primary-video",
+ producer: generationProducers.primaryVideo,
+ inputs: { set: operation("generate") },
+ result: { kind: "output", name: "video" },
+ },
+ );
+ const shape = mediaInputs.map((item) => `${item.name}=${item.port}`).join(",") || "no-media";
return sealGraphFragment({
- name: `@narratage/seedance/${endpoint.key}-speech-primary-video@1`,
- inputs: [
- { name: "program", type: { module: endpoint.producer.module, name: "SpeechProgram" } },
- { name: "duration", type: speechTypes.duration },
- ],
- operations: [
- {
- id: "compile-request",
- producer: compileProducer,
- inputs: { program: input("program"), duration: input("duration") },
- result: { kind: "output", name: "request" },
- },
- {
- id: "generate",
- producer: endpoint.producer,
- inputs: { request: operation("compile-request") },
- result: { kind: "need", name: "generation", accepts: "exact" },
- },
- {
- id: "select-primary-video",
- producer: generationProducers.primaryVideo,
- inputs: { set: operation("generate") },
- result: { kind: "output", name: "video" },
- },
- ],
+ name: `@narratage/seedance/${endpoint.key}-speech-assembled-primary-video[${shape}]@1`,
+ inputs,
+ operations,
exports: [{
name: "video",
type: artifactTypes.blob,
root: operation("select-primary-video"),
- semanticInputs: ["program", "duration"],
+ semanticInputs: inputs.map((entry) => entry.name),
fidelity: "exact",
}],
});
diff --git a/packages/seedance/src/index.ts b/packages/seedance/src/index.ts
index a85b70ad..f88a8d91 100644
--- a/packages/seedance/src/index.ts
+++ b/packages/seedance/src/index.ts
@@ -1,9 +1,9 @@
-import { artifactDependency } from "@narratage/artifact";
import {
generationObjectSchema,
generationPromptSchema,
portsObjectSchema,
sealGenerationPortRequest,
+ sealGenerationRequestDraft,
sealGenerationPortTable,
verifyPortsAgainstTable,
} from "@narratage/generation";
@@ -11,6 +11,7 @@ import type {
GenerationPortTable,
GenerationPortValue,
GenerationRequest,
+ GenerationRequestDraft,
} from "@narratage/generation";
import { defineExactModelModule } from "@narratage/model-kit";
import { narrativeDependency } from "@narratage/narrative";
@@ -149,7 +150,8 @@ export function verifySeedanceSpeechProgram(value: unknown): asserts value is Se
throw new Error("Seedance SpeechProgram identity is invalid");
}
const table = seedancePorts[value.model as SeedanceModel];
- verifyPortsAgainstTable(table, value.ports, { omit: ["duration"] });
+ const media = table.ports.filter((port) => port.value.kind === "media").map((port) => port.name);
+ verifyPortsAgainstTable(table, value.ports, { omit: ["duration", ...media] });
const generateAudio = (value.ports as SeedancePortMap).generateAudio;
if (generateAudio?.[0] !== true) throw new Error("Seedance SpeechProgram must generate audio");
}
@@ -160,19 +162,26 @@ export function sealSeedanceSpeechProgram(value: SeedanceSpeechProgram): Seedanc
return result;
}
-export function compileSeedanceSpeechRequest(
+export function compileSeedanceSpeechRequestDraft(
program: SeedanceSpeechProgram,
duration: SpeechDuration,
-): GenerationRequest {
+): GenerationRequestDraft {
verifySeedanceSpeechProgram(program);
assertSpeechDurationIdentity(duration);
- return sealSeedanceRequest(program.model, { ...program.ports, duration: [duration.durationSec] });
+ return sealGenerationRequestDraft(seedancePorts[program.model], {
+ ...program.ports,
+ duration: [duration.durationSec],
+ });
}
const speechSpineSchema = (model: SeedanceModel): ValueSchema => generationObjectSchema({
contract: { schema: { kind: "literal", value: "svml.seedance-speech-spine@1" } },
model: { schema: { kind: "literal", value: model } },
- ports: { schema: portsObjectSchema(seedancePorts[model], { omit: ["duration"] }) },
+ ports: {
+ schema: portsObjectSchema(seedancePorts[model], {
+ omit: ["duration", ...seedancePorts[model].ports.filter((port) => port.value.kind === "media").map((port) => port.name)],
+ }),
+ },
});
const seedanceBaseDefinition = defineExactModelModule({
@@ -200,7 +209,6 @@ export const seedanceManifest = {
...seedanceBaseDefinition.manifest,
dependencies: [
...seedanceBaseDefinition.manifest.dependencies,
- artifactDependency,
narrativeDependency,
speechDependency,
],
@@ -234,7 +242,13 @@ export const seedanceManifest = {
name: "speech",
tag: "Speech",
mode: "structured",
- outputs: [seedanceTypes.speechSpine, ...Object.values(seedanceEndpoints).map((endpoint) => endpoint.requestType)],
+ outputs: [
+ seedanceTypes.speechSpine,
+ ...Object.values(seedanceEndpoints).flatMap((endpoint) => [
+ endpoint.draftType,
+ ...Object.values(endpoint.mediaBindings).map((binding) => binding.type),
+ ]),
+ ],
implementation: {
kind: "trusted-frontend-surface",
locator: "@narratage/seedance/speech-surface",
@@ -245,7 +259,10 @@ export const seedanceManifest = {
name: "video",
tag: "Video",
mode: "structured",
- outputs: Object.values(seedanceEndpoints).map((endpoint) => endpoint.requestType),
+ outputs: Object.values(seedanceEndpoints).flatMap((endpoint) => [
+ endpoint.draftType,
+ ...Object.values(endpoint.mediaBindings).map((binding) => binding.type),
+ ]),
implementation: {
kind: "trusted-frontend-surface",
locator: "@narratage/seedance/video-surface",
@@ -261,7 +278,7 @@ export const seedanceManifest = {
{ name: "program", type: seedanceTypes.speechSpine },
{ name: "duration", type: speechTypes.duration },
],
- outputs: [{ name: "request", type: seedanceEndpointsByModel[model].requestType }],
+ outputs: [{ name: "draft", type: seedanceEndpointsByModel[model].draftType }],
needs: [],
implementation: {
kind: "registered" as const,
@@ -281,9 +298,9 @@ export const seedanceComponent = {
implementationDigest: seedanceSpeechCompileImplementationDigests[model],
handler: ({ inputs }: { readonly inputs: Readonly<Record<string, { readonly value: import("@narratage/protocol").StoredValue }>> }) => ({
outputs: {
- request: {
+ draft: {
kind: "inline" as const,
- value: canonicalize(compileSeedanceSpeechRequest(
+ value: canonicalize(compileSeedanceSpeechRequestDraft(
inputs.program?.value.kind === "inline"
? inputs.program.value.value as unknown as SeedanceSpeechProgram
: (() => { throw new Error("Seedance SpeechProgram must be inline"); })(),
@@ -305,6 +322,7 @@ export const seedanceDefinition = {
};
export {
+ createSeedanceAssembledGenerationFragment,
createSeedanceGenerationFragment,
createSeedanceSpeechGenerationFragment,
} from "./fragment.js";
diff --git a/packages/seedance/src/surface.ts b/packages/seedance/src/surface.ts
index 93a2bf42..fe6b13b1 100644
--- a/packages/seedance/src/surface.ts
+++ b/packages/seedance/src/surface.ts
@@ -3,7 +3,7 @@ import type { NarrativeDialogueExcerpt } from "@narratage/narrative";
import { speechTypes } from "@narratage/speech";
import type { SpeechDuration } from "@narratage/speech";
import { artifactTypes } from "@narratage/artifact";
-import type { BlobRef, CanonicalValue } from "@narratage/protocol";
+import type { CanonicalValue } from "@narratage/protocol";
import type {
StructuredElement,
StructuredSurfaceHandler,
@@ -12,15 +12,20 @@ import type {
} from "@narratage/text";
import {
- createSeedanceGenerationFragment,
+ createSeedanceAssembledGenerationFragment,
createSeedanceSpeechGenerationFragment,
} from "./fragment.js";
-import { generationPort } from "@narratage/generation";
-import type { GenerationMediaRole, GenerationPortTable } from "@narratage/generation";
+import {
+ generationPort,
+ sealGenerationMediaBinding,
+ sealGenerationRequestDraft,
+} from "@narratage/generation";
+import type { GenerationMediaPort, GenerationMediaRole, GenerationPortTable } from "@narratage/generation";
+import { exactModelMediaInputNames } from "@narratage/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput } from "@narratage/model-kit";
import {
sealSeedancePrompt,
- sealSeedanceRequest,
sealSeedanceSpeechProgram,
seedanceEndpoints,
seedancePorts,
@@ -37,7 +42,7 @@ import type {
/** `role` is the media kind the port accepts; `label` is the author's own note about it. */
type ReferenceInput = {
readonly role: GenerationMediaRole;
- readonly artifact: BlobRef;
+ readonly source: SurfaceResolvedReference;
readonly label?: string;
};
@@ -115,11 +120,13 @@ function prompt(reference: SurfaceResolvedReference, subject: string): SeedanceP
return value;
}
-function blob(reference: SurfaceResolvedReference, subject: string): BlobRef {
+function mediaReference(reference: SurfaceResolvedReference, role: GenerationMediaRole, subject: string): SurfaceResolvedReference {
if (!sameType(reference.type, artifactTypes.blob)) throw new Error(`${subject} must reference a BlobArtifact`);
const value = reference.record?.value;
- if (value?.kind !== "blob") throw new Error(`${subject} must reference an authored artifact declared before use`);
- return value;
+ if (value !== undefined && (value.kind !== "blob" || !value.mediaType.startsWith(`${role}/`))) {
+ throw new Error(`${subject} must reference ${role} media`);
+ }
+ return reference;
}
function normalizedText(element: StructuredElement): string {
@@ -225,13 +232,9 @@ function references(
const kinds = accepted.filter((kind) => child.attributes[kind] !== undefined);
if (kinds.length !== 1) throw new Error(`${child.name} requires exactly one of ${accepted.join(", ")}`);
const role = kinds[0]!;
- const source = resolved(child, role, resolveReference);
- const artifact = blob(source, `${child.name}.${role}`);
- if (!artifact.mediaType.startsWith(`${role}/`)) {
- throw new Error(`${child.name}.${role} must reference ${role} media`);
- }
+ const source = mediaReference(resolved(child, role, resolveReference), role, `${child.name}.${role}`);
const label = optionalStringAttribute(child, "role");
- result.push({ role, artifact, ...(label === undefined ? {} : { label }) });
+ result.push({ role, source, ...(label === undefined ? {} : { label }) });
}
// Every bound below is read from the model's own port table, never repeated here.
for (const role of accepted) {
@@ -251,13 +254,49 @@ function referencePrompt(values: readonly ReferenceInput[]): string {
return roles.length === 0 ? "" : `\n\nReference roles:\n${roles.join("\n")}`;
}
-/** Authored references split into the model's per-modality ports; labels never reach the wire. */
-function referencePorts(values: readonly ReferenceInput[]): SeedancePortMap {
- return Object.fromEntries((Object.keys(REFERENCE_PORTS) as readonly GenerationMediaRole[])
- .map((role) => [REFERENCE_PORTS[role], values
- .filter((item) => item.role === role)
- .map(({ artifact }) => ({ role, artifact }))])
- .filter(([, items]) => (items as readonly unknown[]).length > 0));
+type AssembledReferences = {
+ readonly mediaInputs: readonly ExactModelMediaInput[];
+ readonly records: readonly {
+ readonly id: string;
+ readonly type: SurfaceResolvedReference["type"];
+ readonly value: { readonly kind: "inline"; readonly value: CanonicalValue };
+ readonly range: StructuredElement["range"];
+ }[];
+ readonly inputs: Readonly<Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }>>;
+};
+
+function assembleReferences(
+ id: string,
+ endpoint: ExactModelEndpoint,
+ values: readonly ReferenceInput[],
+ range: StructuredElement["range"],
+): AssembledReferences {
+ const records: AssembledReferences["records"][number][] = [];
+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }> = {};
+ const mediaInputs = values.map((value, index): ExactModelMediaInput => {
+ const name = `media-${String(index + 1).padStart(4, "0")}`;
+ const portName = REFERENCE_PORTS[value.role];
+ const binding = endpoint.mediaBindings[portName];
+ if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${portName}`);
+ const port = generationPort(endpoint.ports, portName);
+ if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${portName} is not media`);
+ const mediaPort = port as GenerationMediaPort;
+ const bindingId = `${id}.${name}.binding`;
+ records.push({
+ id: bindingId,
+ type: binding.type,
+ value: {
+ kind: "inline",
+ value: sealGenerationMediaBinding(mediaPort, { role: value.role }) as unknown as CanonicalValue,
+ },
+ range,
+ });
+ const names = exactModelMediaInputNames(name);
+ inputs[names.binding] = { kind: "record", id: bindingId };
+ inputs[names.artifact] = value.source.ref;
+ return { name, port: portName };
+ });
+ return { mediaInputs, records, inputs };
}
function dialogueExcerpt(reference: SurfaceResolvedReference, subject: string): { readonly dialogue: string } {
@@ -296,24 +335,26 @@ function speechDurationReference(
function generationOutput(
element: StructuredElement,
- request: ReturnType<typeof sealSeedanceRequest>,
+ endpoint: ExactModelEndpoint,
+ draft: ReturnType<typeof sealGenerationRequestDraft>,
+ references: readonly ReferenceInput[],
output: string,
) {
- const { endpoint } = modelSelection(element);
- const fragment = createSeedanceGenerationFragment(endpoint);
- const requestId = `${stringAttribute(element, "id")}.request`;
const id = stringAttribute(element, "id");
+ const assembled = assembleReferences(id, endpoint, references, element.range);
+ const fragment = createSeedanceAssembledGenerationFragment(endpoint, assembled.mediaInputs);
+ const draftId = `${id}.draft`;
return {
records: [{
- id: requestId,
- type: endpoint.requestType,
- value: { kind: "inline" as const, value: request as unknown as CanonicalValue },
+ id: draftId,
+ type: endpoint.draftType,
+ value: { kind: "inline" as const, value: draft as unknown as CanonicalValue },
range: element.range,
- }],
+ }, ...assembled.records],
components: [{
id,
fragment: fragment.id,
- inputs: { request: { kind: "record" as const, id: requestId } },
+ inputs: { draft: { kind: "record" as const, id: draftId }, ...assembled.inputs },
outputs: { video: output },
range: element.range,
}],
@@ -344,13 +385,12 @@ export const decodeSeedanceVideoSurface: StructuredSurfaceHandler = ({ element,
const selected = modelSelection(element);
const declaredPrompt = prompt(resolved(element, "prompt", resolveReference), `${element.name}.prompt`);
const refs = references(element, selected.model, resolveReference);
- const request = sealSeedanceRequest(selected.model, {
+ const draft = sealGenerationRequestDraft(seedancePorts[selected.model], {
prompt: [declaredPrompt.text + referencePrompt(refs)],
- ...referencePorts(refs),
...generationSettings(element, selected.model),
generateAudio: [booleanAttribute(element, "generate-audio", false)],
});
- return generationOutput(element, request, `${stringAttribute(element, "id")}.video`);
+ return generationOutput(element, selected.endpoint, draft, refs, `${stringAttribute(element, "id")}.video`);
};
export const decodeSeedanceSpeechSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
@@ -370,16 +410,17 @@ export const decodeSeedanceSpeechSurface: StructuredSurfaceHandler = ({ element,
model: selected.model,
ports: {
prompt: [promptText],
- ...referencePorts(refs),
...settings,
generateAudio: [true],
},
});
const id = stringAttribute(element, "id");
const programId = `${id}.program`;
+ const assembled = assembleReferences(id, selected.endpoint, refs, element.range);
const fragment = createSeedanceSpeechGenerationFragment(
selected.endpoint,
seedanceSpeechCompileProducers[selected.model],
+ assembled.mediaInputs,
);
return {
records: [{
@@ -387,13 +428,14 @@ export const decodeSeedanceSpeechSurface: StructuredSurfaceHandler = ({ element,
type: seedanceTypes.speechSpine,
value: { kind: "inline", value: program as unknown as CanonicalValue },
range: element.range,
- }],
+ }, ...assembled.records],
components: [{
id,
fragment: fragment.id,
inputs: {
program: { kind: "record", id: programId },
duration: duration.ref,
+ ...assembled.inputs,
},
outputs: { video: id },
range: element.range,
@@ -401,11 +443,10 @@ export const decodeSeedanceSpeechSurface: StructuredSurfaceHandler = ({ element,
fragments: [fragment],
};
}
- const request = sealSeedanceRequest(selected.model, {
+ const draft = sealGenerationRequestDraft(seedancePorts[selected.model], {
prompt: [promptText],
- ...referencePorts(refs),
...generationSettings(element, selected.model),
generateAudio: [true],
});
- return generationOutput(element, request, stringAttribute(element, "id"));
+ return generationOutput(element, selected.endpoint, draft, refs, stringAttribute(element, "id"));
};
diff --git a/packages/seedance/test/surface.test.ts b/packages/seedance/test/surface.test.ts
new file mode 100644
index 00000000..39f37a8d
--- /dev/null
+++ b/packages/seedance/test/surface.test.ts
@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { artifactTypes } from "@narratage/artifact";
+import { digestOf } from "@narratage/protocol";
+import { parseStructuredElement } from "@narratage/text";
+import type { SurfaceResolvedReference } from "@narratage/text";
+
+import {
+ decodeSeedanceVideoSurface,
+ sealSeedancePrompt,
+ seedanceTypes,
+} from "../src/index.js";
+
+test("Seedance keeps an upstream generated image as a graph edge", async () => {
+ const element = parseStructuredElement({
+ name: "dynamic-reference.svml",
+ text: `<seedance:Video id="motion" model="mini" prompt={direction} duration="6" resolution="720p" aspect-ratio="9:16">
+ <seedance:Reference image={generated.image} role="montage reference"/>
+ </seedance:Video>`,
+ }, 0).element;
+ const prompt = sealSeedancePrompt("Create a coherent montage.");
+ const refs = new Map<string, SurfaceResolvedReference>([
+ ["direction", {
+ path: "direction",
+ ref: { kind: "record", id: "direction" },
+ type: seedanceTypes.prompt,
+ record: {
+ id: "direction",
+ type: seedanceTypes.prompt,
+ value: { kind: "inline", value: prompt },
+ digest: digestOf(prompt),
+ conformance: "exact",
+ origin: { kind: "authored", sourceDigest: digestOf("source"), frontendClosureDigest: digestOf("frontend") },
+ },
+ }],
+ ["generated.image", {
+ path: "generated.image",
+ ref: { kind: "component-output", component: "generated", output: "image" },
+ type: artifactTypes.blob,
+ }],
+ ]);
+ const result = await decodeSeedanceVideoSurface({
+ sourceName: "dynamic-reference.svml",
+ element,
+ resolveReference: (path) => refs.get(path),
+ resolveAsset: () => { throw new Error("no asset"); },
+ });
+ const component = result.components[0]!;
+ assert.deepEqual(component.inputs["media-0001:artifact"], {
+ kind: "component-output", component: "generated", output: "image",
+ });
+ assert.equal(component.inputs["media-0001:binding"]?.kind, "record");
+ const draft = result.records.find((record) => record.id === "motion.draft");
+ assert.ok(draft?.value.kind === "inline");
+ assert.equal(JSON.stringify(draft.value).includes("referenceImage"), false);
+});
diff --git a/packages/video-cli/test/cli.test.ts b/packages/video-cli/test/cli.test.ts
index 43973436..4edea6a2 100644
--- a/packages/video-cli/test/cli.test.ts
+++ b/packages/video-cli/test/cli.test.ts
@@ -235,8 +235,10 @@ test("Prompt Kit source and Speaker Surface finish prompt assembly before the Ru
readonly steps: readonly { readonly producer: { readonly name: string } }[];
};
assert.deepEqual(plan.steps.map((step) => step.producer.name).sort(), [
+ "bind-request-seedance-2-mini-referenceImage",
"compile-seedance-2-mini-speech-request",
"estimate-speech-duration",
+ "finalize-request-seedance-2-mini",
"request-seedance-2-mini",
"select-primary-video",
].sort());
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index b88628ec..7142e75f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1114,9 +1114,18 @@ importers:
packages/gpt-image:
dependencies:
+ '@narratage/artifact':
+ specifier: workspace:*
+ version: link:../artifact
+ '@narratage/elaborator':
+ specifier: workspace:*
+ version: link:../elaborator
'@narratage/generation':
specifier: workspace:*
version: link:../generation
+ '@narratage/image-transform':
+ specifier: workspace:*
+ version: link:../image-transform
'@narratage/model-kit':
specifier: workspace:*
version: link:../model-kit