Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix: finish HypiHub reference and audio mappings
rponee58e20bf0518
diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.mdindex b243e719..2855fd53 100644--- a/packages/provider-hypihub/README.md+++ b/packages/provider-hypihub/README.md@@ -6,10 +6,10 @@ when its key is configured.It maps the currently shipped image/video model capabilities to HypiHub, including image edits andimage-to-video first-frame inputs, submits jobs, polls them, downloads the first-class assets and-persists them in Hypit's configured ArtifactStore. Image references are sent as HypiHub-compatible-`data:` URLs. Video-reference inputs require a public HTTPS URL and are rejected clearly when the-configured ArtifactStore cannot provide one. It also exports a small Gemini-native VLM generator for-callers that previously used Vertex.+persists them in Hypit's configured ArtifactStore. Image references use HypiHub's `reference_images`+shape; video reference arrays use the documented `extra` passthrough so HypiHub can hand them to its+selected upstream adaptor. First/last-frame images use `input_reference` and `last_frame`. It also+exports a small Gemini-native VLM generator for callers that previously used Vertex.Runtime Profile example:diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.tsindex 55bfc64d..6c04096c 100644--- a/packages/provider-hypihub/src/mapping.ts+++ b/packages/provider-hypihub/src/mapping.ts@@ -8,16 +8,17 @@ const NANO_BANANA: ModuleRef = { name: "@hypit/nano-banana", version: "1" };const SEEDREAM: ModuleRef = { name: "@hypit/seedream", version: "1" };const MINIMAX: ModuleRef = { name: "@hypit/minimax-h3", version: "1" };const GROK: ModuleRef = { name: "@hypit/grok-imagine", version: "1" };+const MIMO_TTS: ModuleRef = { name: "@hypit/mimo-tts", version: "1" };const seedance = (name: string, model: string): GenerationWireMapping => ({capability: { module: SEEDANCE, name }, result: "video", routes: [{ model }],fields: {prompt: { as: "value", field: "prompt" },- referenceImage: { as: "urlArray", field: "reference_image_urls" },- referenceVideo: { as: "urlArray", field: "reference_videos" },+ referenceImage: { as: "urlArray", field: "reference_image_urls" },+ referenceVideo: { as: "urlArray", field: "reference_videos" },referenceAudio: { as: "urlArray", field: "reference_audios" },- firstFrame: { as: "url", field: "first_image_url" },- lastFrame: { as: "url", field: "last_image_url" },+ firstFrame: { as: "url", field: "first_image_url" },+ lastFrame: { as: "url", field: "last_image_url" },resolution: { as: "value", field: "resolution" },aspectRatio: { as: "value", field: "aspect_ratio" },duration: { as: "value", field: "seconds" },@@ -40,7 +41,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [prompt: { as: "value", field: "prompt" },aspectRatio: { as: "value", field: "aspect_ratio" },resolution: { as: "value", field: "size", whenAbsent: "1024x1024" },- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },+ images: { as: "urlArray", field: "reference_images" },},},...(["nano-banana-2", "nano-banana-pro"] as const).map((model) => ({@@ -49,7 +50,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [prompt: { as: "value" as const, field: "prompt" },aspectRatio: { as: "value" as const, field: "aspect_ratio" },resolution: { as: "value" as const, field: "size", whenAbsent: "1024x1024" },- images: { as: "itemObject" as const, field: "reference_images", urlKey: "url", fieldKeys: {} },+ images: { as: "urlArray" as const, field: "reference_images" },outputFormat: { as: "value" as const, field: "output_format" },},})),@@ -62,7 +63,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [prompt: { as: "value", field: "prompt" },aspectRatio: { as: "value", field: "aspect_ratio" },resolution: { as: "value", field: "size", whenAbsent: "2048x2048" },- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },+ images: { as: "urlArray", field: "reference_images" },quality: { as: "value", field: "quality" },outputFormat: { as: "value", field: "output_format" },nsfwCheck: { as: "value", field: "nsfw_checker" },@@ -77,9 +78,9 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [],fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },- resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },- referenceImage: { as: "urlArray", field: "reference_image_urls" },- referenceVideo: { as: "urlArray", field: "reference_videos" },+ resolution: { as: "value", field: "resolution", whenAbsent: "2k" }, aspectRatio: { as: "value", field: "aspect_ratio" },+ referenceImage: { as: "urlArray", field: "reference_image_urls" },+ referenceVideo: { as: "urlArray", field: "reference_videos" },referenceAudio: { as: "urlArray", field: "reference_audios" },firstFrame: { as: "url", field: "first_image_url" },lastFrame: { as: "url", field: "last_image_url" },@@ -93,7 +94,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },+ images: { as: "urlArray", field: "reference_images" },sourceTaskId: { as: "value", field: "source_task_id" },},},@@ -105,7 +106,30 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },+ images: { as: "urlArray", field: "reference_images" },+ },+ },+ {+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts" }, result: "audio", routes: [{ model: "mimo-v2.5-tts" }],+ fields: {+ text: { as: "value", field: "input" },+ instruction: { as: "value", field: "prompt" },+ voice: { as: "value", field: "voice" },+ },+ },+ {+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts-voicedesign" }, result: "audio", routes: [{ model: "mimo-v2.5-tts-voicedesign" }],+ fields: {+ text: { as: "value", field: "input" },+ voiceDescription: { as: "value", field: "voice_description" },+ },+ },+ {+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts-voiceclone" }, result: "audio", routes: [{ model: "mimo-v2.5-tts-voiceclone" }],+ fields: {+ text: { as: "value", field: "input" },+ instruction: { as: "value", field: "prompt" },+ sample: { as: "url", field: "voice" },},},];diff --git a/packages/provider-hypihub/src/provider.ts b/packages/provider-hypihub/src/provider.tsindex f43642f7..99fcd978 100644--- a/packages/provider-hypihub/src/provider.ts+++ b/packages/provider-hypihub/src/provider.ts@@ -17,6 +17,8 @@ export type CreateHypiHubProviderOptions = {readonly pollIntervalMs?: number;readonly requestTimeoutMs?: number;readonly fetch?: typeof globalThis.fetch;+ /** Uploads a Runtime artifact to a public URL for video/audio references. */+ readonly publicAssetUrl?: (artifact: BlobRef, artifacts: ArtifactStore) => Promise<string>;};type Handle = { readonly contract: "hypit.hypihub-operation@1"; readonly jobId: string; readonly route: string; readonly startedAt: number };@@ -41,7 +43,7 @@ function jobId(value: Record<string, unknown>): string {return id;}-async function verifyModelRoute(client: HypiHubClient, apiKey: string, model: string, operation: "images" | "image_edits" | "videos"): Promise<void> {+async function verifyModelRoute(client: HypiHubClient, apiKey: string, model: string, operation: "images" | "image_edits" | "videos" | "audio_speech"): Promise<void> {const card = await client.json(`/models/${encodeURIComponent(model)}`, apiKey);const endpoints = card.endpoints;assert(Array.isArray(endpoints) && endpoints.includes(operation),@@ -52,7 +54,20 @@ function dataUrl(bytes: Uint8Array, mediaType: string): string {return `data:${mediaType};base64,${Buffer.from(bytes).toString("base64")}`;}-async function resolveArtifactAsDataUrl(artifacts: ArtifactStore, artifact: BlobRef): Promise<string> {+async function resolveArtifactAsDataUrl(+ artifacts: ArtifactStore,+ artifact: BlobRef,+ publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"],+ allowBinaryInline: boolean,+): Promise<string> {+ if (!allowBinaryInline && /^(?:video|audio)\//u.test(artifact.mediaType)) {+ if (publicAssetUrl !== undefined) {+ const url = (await publicAssetUrl(artifact, artifacts)).trim();+ assert(/^https:\/\//iu.test(url), "publicAssetUrl must return an https URL for HypiHub references");+ return url;+ }+ throw new Error("HypiHub video/audio references require a public HTTPS URL; configure publicAssetUrl on the provider");+ }const bytes = await artifacts.get(artifact.digest);assert(bytes !== undefined, `HypiHub reference artifact ${artifact.digest} is unavailable`);return dataUrl(bytes, artifact.mediaType);@@ -93,24 +108,25 @@ async function complete(client: HypiHubClient, apiKey: string, route: (typeof hyreturn { status: "completed", result: { value: route.packageResult(blobs) } };}-async function synthesizeAudio(client: HypiHubClient, context: EndpointInvocationContext): Promise<EndpointFulfillment> {+async function synthesizeAudio(client: HypiHubClient, context: EndpointInvocationContext, publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"]): Promise<EndpointFulfillment> {const route = hypiHubRouteForCapability(context.need.capability);assert(route !== undefined && route.media === "audio", "HypiHub does not implement this exact capability");const compiled = await route.compile(context.need.constraints,- (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact, publicAssetUrl, true));+ await verifyModelRoute(client, credential(context), compiled.model, "audio_speech");const audio = await client.binary("/audio/speech", credential(context), { model: compiled.model, ...(compiled.input as Record<string, unknown>) });const artifact = await context.artifacts.put(audio.bytes, audio.mediaType);return { value: route.packageResult([artifact]) };}-function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs: number): AsyncEndpoint {+function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs: number, publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"]): AsyncEndpoint {return {async start(context: EndpointStartContext) {try {const route = hypiHubRouteForCapability(context.need.capability);assert(route !== undefined, "HypiHub does not implement this exact capability"); const apiKey = credential(context);const compiled = await route.compile(context.need.constraints,- (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact, publicAssetUrl, false));assert(route.media !== "audio", "HypiHub audio capabilities use an immediate endpoint");const input = compiled.input as Record<string, unknown>;const hasReferences = Object.entries(input).some(([key, value]) => {@@ -145,8 +161,8 @@ function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs:export function createHypiHubProvider(options: CreateHypiHubProviderOptions = {}) {const client = new HypiHubClient({ baseUrl: options.baseUrl ?? "https://hypit.ai/v1", timeout: options.requestTimeoutMs ?? 30_000, fetcher: options.fetch ?? globalThis.fetch });- const asyncEndpoint = endpoint(client, options.pollIntervalMs ?? 5_000, 20 * 60_000);- const audioEndpoint: ImmediateEndpointHandler = (context) => synthesizeAudio(client, context);+ const asyncEndpoint = endpoint(client, options.pollIntervalMs ?? 5_000, 20 * 60_000, options.publicAssetUrl);+ const audioEndpoint: ImmediateEndpointHandler = (context) => synthesizeAudio(client, context, options.publicAssetUrl);return defineEndpointPackage({module: hypiHubProviderModuleRef, facet: "gateway", instance: options.instance ?? "hypihub.default", pool: options.pool ?? options.instance ?? "hypihub.default",credentials: { apiKey: options.apiKey ?? credentialRef("env", "HYPIHUB_API_KEY") }, credentialInputs: { apiKey: { label: "HypiHub API key" } }, defaultConcurrency: options.defaultConcurrency ?? 4,diff --git a/packages/provider-hypihub/src/routes.ts b/packages/provider-hypihub/src/routes.tsindex 0f9430d3..cd0aee5d 100644--- a/packages/provider-hypihub/src/routes.ts+++ b/packages/provider-hypihub/src/routes.ts@@ -52,6 +52,10 @@ function normalizeHypiHubRequest(return { model: request.model, input: canonicalize(input) };}+ // Audio mappings already use HypiHub's public /audio/speech vocabulary.+ // They must not fall through the video reference normalizer below.+ if (mapping.result === "audio") return { model: request.model, input: canonicalize(input) };+if (typeof input.resolution === "string") input.resolution = input.resolution.toLowerCase();const imageRefs = input.reference_image_urls;@@ -61,21 +65,24 @@ function normalizeHypiHubRequest(const lastFrame = input.last_image_url;const audioRefs = input.reference_audios;const sourceTaskId = input.source_task_id;- if (Array.isArray(audioRefs) && audioRefs.length > 0) {- throw new Error("HypiHub unified video API does not support reference audio inputs");- }- if (Array.isArray(videoRefs) && videoRefs.length > 0) {- throw new Error("HypiHub video reference inputs require a public HTTPS URL; the Runtime ArtifactStore cannot expose one");- }- if (Array.isArray(imageRefs) && imageRefs.length > 1) {- throw new Error("HypiHub unified video API accepts one input_reference image; multiple reference images are not supported");- }- if (Array.isArray(genericImageRefs) && genericImageRefs.length > 1) {- throw new Error("HypiHub unified video API accepts one input_reference image; multiple reference images are not supported");- }if (typeof sourceTaskId === "string" && sourceTaskId.length > 0) {throw new Error("HypiHub unified video API does not support video continuation sourceTaskId");}++ // HypiHub's upstream relay exposes vendor-specific reference arrays through+ // its documented `extra` passthrough. Unknown top-level fields are ignored+ // by the public JSON decoder, while `extra` is handed to the selected+ // upstream adaptor and becomes its provider-neutral reference vocabulary.+ const extra = input.extra !== null && typeof input.extra === "object" && !Array.isArray(input.extra)+ ? { ...(input.extra as Record<string, unknown>) } : {};+ if (Array.isArray(imageRefs) && imageRefs.length > 0) extra.reference_image_urls = imageRefs;+ if (Array.isArray(genericImageRefs) && genericImageRefs.length > 0) {+ const urls = genericImageRefs.map((item) => item !== null && typeof item === "object"+ ? (item as Record<string, unknown>).url : item).filter((item): item is string => typeof item === "string" && item.length > 0);+ if (urls.length > 0) extra.reference_image_urls = urls;+ }+ if (Array.isArray(videoRefs) && videoRefs.length > 0) extra.reference_videos = videoRefs;+ if (Array.isArray(audioRefs) && audioRefs.length > 0) extra.reference_audios = audioRefs;delete input.reference_image_urls;delete input.reference_images;delete input.reference_videos;@@ -83,15 +90,10 @@ function normalizeHypiHubRequest(delete input.first_image_url;delete input.last_image_url;delete input.source_task_id;+ delete input.extra;if (typeof firstFrame === "string" && firstFrame.length > 0) input.input_reference = firstFrame;- else if (Array.isArray(imageRefs) && imageRefs.length > 0) input.input_reference = imageRefs[0];- else if (Array.isArray(genericImageRefs) && genericImageRefs.length > 0) {- const first = genericImageRefs[0];- if (first !== null && typeof first === "object" && typeof (first as Record<string, unknown>).url === "string") {- input.input_reference = (first as Record<string, unknown>).url;- }- }if (typeof lastFrame === "string" && lastFrame.length > 0) input.last_frame = lastFrame;+ if (Object.keys(extra).length > 0) input.extra = extra;return { model: request.model, input: canonicalize(input) };}diff --git a/packages/provider-hypihub/test/routes.test.ts b/packages/provider-hypihub/test/routes.test.tsindex 90392cb1..4c14fb6e 100644--- a/packages/provider-hypihub/test/routes.test.ts+++ b/packages/provider-hypihub/test/routes.test.ts@@ -11,6 +11,7 @@ const image = {};const resolve = async () => "data:image/png;base64,AQID";+const resolveAudio = async () => "data:audio/wav;base64,AQID";test("HypiHub GPT image requests use canonical edit references and size dimensions", async () => {const route = hypiHubRoutes.find((item) => item.capability.name === "gpt-image-2");@@ -28,7 +29,7 @@ test("HypiHub GPT image requests use canonical edit references and size dimensioprompt: "edit",aspect_ratio: "1:1",size: "1024x1024",- reference_images: [{ url: "data:image/png;base64,AQID" }],+ reference_images: ["data:image/png;base64,AQID"],});});@@ -46,18 +47,77 @@ test("HypiHub image-to-video requests use input_reference", async () => {assert.deepEqual(result.input, {prompt: "animate",seconds: 5,+ resolution: "2k",input_reference: "data:image/png;base64,AQID",});});-test("HypiHub rejects reference video/audio that cannot be represented by its public API", async () => {+test("HypiHub preserves video/audio reference arrays for its upstream passthrough", async () => {const route = hypiHubRoutes.find((item) => item.capability.name === "minimax-h3");assert.ok(route);- await assert.rejects(() => route.compile({+ const result = await route.compile({ports: {prompt: ["animate"],duration: [5],referenceAudio: [{ role: "audio", artifact: { ...image, mediaType: "audio/wav" } }],},- }, resolve), /does not support reference audio/);+ }, resolveAudio);+ assert.deepEqual(result.input, {+ prompt: "animate", seconds: 5, resolution: "2k",+ extra: { reference_audios: ["data:audio/wav;base64,AQID"] },+ });+});++test("HypiHub Seedance multimodal references ride in the public extra passthrough", async () => {+ const route = hypiHubRoutes.find((item) => item.capability.name === "seedance-2");+ assert.ok(route);+ const result = await route.compile({+ ports: {+ prompt: ["animate"],+ duration: [5],+ resolution: ["720p"],+ aspectRatio: ["16:9"],+ generateAudio: [true],+ webSearch: [false],+ referenceImage: [+ { role: "image", artifact: image },+ { role: "image", artifact: { ...image, digest: `sha256:${"2".repeat(64)}` } },+ ],+ },+ }, resolve);+ assert.equal(result.model, "bytedance/seedance-2");+ assert.deepEqual(result.input, {+ prompt: "animate",+ seconds: 5,+ resolution: "720p",+ aspect_ratio: "16:9",+ generate_audio: true,+ web_search: false,+ extra: { reference_image_urls: ["data:image/png;base64,AQID", "data:image/png;base64,AQID"] },+ });+});++test("HypiHub MiniMax reference mode preserves image arrays in extra", async () => {+ const route = hypiHubRoutes.find((item) => item.capability.name === "minimax-h3");+ assert.ok(route);+ const result = await route.compile({ ports: {+ prompt: ["animate"], duration: [6], aspectRatio: ["16:9"],+ referenceImage: [{ role: "image", artifact: image }],+ } }, resolve);+ assert.equal(result.model, "minimax-h3/reference-to-video");+ assert.deepEqual(result.input, {+ prompt: "animate", seconds: 6, aspect_ratio: "16:9",+ resolution: "2k",+ extra: { reference_image_urls: ["data:image/png;base64,AQID"] },+ });+});++test("HypiHub MiMo TTS mappings use the public audio speech fields", async () => {+ const voiceDesign = hypiHubRoutes.find((item) => item.capability.name === "mimo-v2.5-tts-voicedesign");+ assert.ok(voiceDesign);+ const result = await voiceDesign.compile({+ ports: { text: ["hello"], voiceDescription: ["warm and calm"] },+ }, resolve);+ assert.equal(result.model, "mimo-v2.5-tts-voicedesign");+ assert.deepEqual(result.input, { input: "hello", voice_description: "warm and calm" });});