Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix: finish HypiHub reference and audio mappings

rponee58e20bf0518
diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.md
index b243e719..2855fd53 100644
--- a/packages/provider-hypihub/README.md
+++ b/packages/provider-hypihub/README.md
@@ -6,10 +6,10 @@ when its key is configured.
It maps the currently shipped image/video model capabilities to HypiHub, including image edits and
image-to-video first-frame inputs, submits jobs, polls them, downloads the first-class assets and
-persists them in Hypit's configured ArtifactStore. Image references are sent as HypiHub-compatible
-`data:` URLs. Video-reference inputs require a public HTTPS URL and are rejected clearly when the
-configured ArtifactStore cannot provide one. It also exports a small Gemini-native VLM generator for
-callers that previously used Vertex.
+persists them in Hypit's configured ArtifactStore. Image references use HypiHub's `reference_images`
+shape; video reference arrays use the documented `extra` passthrough so HypiHub can hand them to its
+selected upstream adaptor. First/last-frame images use `input_reference` and `last_frame`. It also
+exports a small Gemini-native VLM generator for callers that previously used Vertex.
Runtime Profile example:
diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.ts
index 55bfc64d..6c04096c 100644
--- a/packages/provider-hypihub/src/mapping.ts
+++ b/packages/provider-hypihub/src/mapping.ts
@@ -8,16 +8,17 @@ const NANO_BANANA: ModuleRef = { name: "@hypit/nano-banana", version: "1" };
const SEEDREAM: ModuleRef = { name: "@hypit/seedream", version: "1" };
const MINIMAX: ModuleRef = { name: "@hypit/minimax-h3", version: "1" };
const GROK: ModuleRef = { name: "@hypit/grok-imagine", version: "1" };
+const MIMO_TTS: ModuleRef = { name: "@hypit/mimo-tts", version: "1" };
const seedance = (name: string, model: string): GenerationWireMapping => ({
capability: { module: SEEDANCE, name }, result: "video", routes: [{ model }],
fields: {
prompt: { as: "value", field: "prompt" },
- referenceImage: { as: "urlArray", field: "reference_image_urls" },
- referenceVideo: { as: "urlArray", field: "reference_videos" },
+ referenceImage: { as: "urlArray", field: "reference_image_urls" },
+ referenceVideo: { as: "urlArray", field: "reference_videos" },
referenceAudio: { as: "urlArray", field: "reference_audios" },
- firstFrame: { as: "url", field: "first_image_url" },
- lastFrame: { as: "url", field: "last_image_url" },
+ firstFrame: { as: "url", field: "first_image_url" },
+ lastFrame: { as: "url", field: "last_image_url" },
resolution: { as: "value", field: "resolution" },
aspectRatio: { as: "value", field: "aspect_ratio" },
duration: { as: "value", field: "seconds" },
@@ -40,7 +41,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
prompt: { as: "value", field: "prompt" },
aspectRatio: { as: "value", field: "aspect_ratio" },
resolution: { as: "value", field: "size", whenAbsent: "1024x1024" },
- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },
+ images: { as: "urlArray", field: "reference_images" },
},
},
...(["nano-banana-2", "nano-banana-pro"] as const).map((model) => ({
@@ -49,7 +50,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
prompt: { as: "value" as const, field: "prompt" },
aspectRatio: { as: "value" as const, field: "aspect_ratio" },
resolution: { as: "value" as const, field: "size", whenAbsent: "1024x1024" },
- images: { as: "itemObject" as const, field: "reference_images", urlKey: "url", fieldKeys: {} },
+ images: { as: "urlArray" as const, field: "reference_images" },
outputFormat: { as: "value" as const, field: "output_format" },
},
})),
@@ -62,7 +63,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
prompt: { as: "value", field: "prompt" },
aspectRatio: { as: "value", field: "aspect_ratio" },
resolution: { as: "value", field: "size", whenAbsent: "2048x2048" },
- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },
+ images: { as: "urlArray", field: "reference_images" },
quality: { as: "value", field: "quality" },
outputFormat: { as: "value", field: "output_format" },
nsfwCheck: { as: "value", field: "nsfw_checker" },
@@ -77,9 +78,9 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
],
fields: {
prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },
- resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },
- referenceImage: { as: "urlArray", field: "reference_image_urls" },
- referenceVideo: { as: "urlArray", field: "reference_videos" },
+ resolution: { as: "value", field: "resolution", whenAbsent: "2k" }, aspectRatio: { as: "value", field: "aspect_ratio" },
+ referenceImage: { as: "urlArray", field: "reference_image_urls" },
+ referenceVideo: { as: "urlArray", field: "reference_videos" },
referenceAudio: { as: "urlArray", field: "reference_audios" },
firstFrame: { as: "url", field: "first_image_url" },
lastFrame: { as: "url", field: "last_image_url" },
@@ -93,7 +94,7 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
fields: {
prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },
resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },
- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },
+ images: { as: "urlArray", field: "reference_images" },
sourceTaskId: { as: "value", field: "source_task_id" },
},
},
@@ -105,7 +106,30 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
fields: {
prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },
resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },
- images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },
+ images: { as: "urlArray", field: "reference_images" },
+ },
+ },
+ {
+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts" }, result: "audio", routes: [{ model: "mimo-v2.5-tts" }],
+ fields: {
+ text: { as: "value", field: "input" },
+ instruction: { as: "value", field: "prompt" },
+ voice: { as: "value", field: "voice" },
+ },
+ },
+ {
+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts-voicedesign" }, result: "audio", routes: [{ model: "mimo-v2.5-tts-voicedesign" }],
+ fields: {
+ text: { as: "value", field: "input" },
+ voiceDescription: { as: "value", field: "voice_description" },
+ },
+ },
+ {
+ capability: { module: MIMO_TTS, name: "mimo-v2.5-tts-voiceclone" }, result: "audio", routes: [{ model: "mimo-v2.5-tts-voiceclone" }],
+ fields: {
+ text: { as: "value", field: "input" },
+ instruction: { as: "value", field: "prompt" },
+ sample: { as: "url", field: "voice" },
},
},
];
diff --git a/packages/provider-hypihub/src/provider.ts b/packages/provider-hypihub/src/provider.ts
index f43642f7..99fcd978 100644
--- a/packages/provider-hypihub/src/provider.ts
+++ b/packages/provider-hypihub/src/provider.ts
@@ -17,6 +17,8 @@ export type CreateHypiHubProviderOptions = {
readonly pollIntervalMs?: number;
readonly requestTimeoutMs?: number;
readonly fetch?: typeof globalThis.fetch;
+ /** Uploads a Runtime artifact to a public URL for video/audio references. */
+ readonly publicAssetUrl?: (artifact: BlobRef, artifacts: ArtifactStore) => Promise<string>;
};
type Handle = { readonly contract: "hypit.hypihub-operation@1"; readonly jobId: string; readonly route: string; readonly startedAt: number };
@@ -41,7 +43,7 @@ function jobId(value: Record<string, unknown>): string {
return id;
}
-async function verifyModelRoute(client: HypiHubClient, apiKey: string, model: string, operation: "images" | "image_edits" | "videos"): Promise<void> {
+async function verifyModelRoute(client: HypiHubClient, apiKey: string, model: string, operation: "images" | "image_edits" | "videos" | "audio_speech"): Promise<void> {
const card = await client.json(`/models/${encodeURIComponent(model)}`, apiKey);
const endpoints = card.endpoints;
assert(Array.isArray(endpoints) && endpoints.includes(operation),
@@ -52,7 +54,20 @@ function dataUrl(bytes: Uint8Array, mediaType: string): string {
return `data:${mediaType};base64,${Buffer.from(bytes).toString("base64")}`;
}
-async function resolveArtifactAsDataUrl(artifacts: ArtifactStore, artifact: BlobRef): Promise<string> {
+async function resolveArtifactAsDataUrl(
+ artifacts: ArtifactStore,
+ artifact: BlobRef,
+ publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"],
+ allowBinaryInline: boolean,
+): Promise<string> {
+ if (!allowBinaryInline && /^(?:video|audio)\//u.test(artifact.mediaType)) {
+ if (publicAssetUrl !== undefined) {
+ const url = (await publicAssetUrl(artifact, artifacts)).trim();
+ assert(/^https:\/\//iu.test(url), "publicAssetUrl must return an https URL for HypiHub references");
+ return url;
+ }
+ throw new Error("HypiHub video/audio references require a public HTTPS URL; configure publicAssetUrl on the provider");
+ }
const bytes = await artifacts.get(artifact.digest);
assert(bytes !== undefined, `HypiHub reference artifact ${artifact.digest} is unavailable`);
return dataUrl(bytes, artifact.mediaType);
@@ -93,24 +108,25 @@ async function complete(client: HypiHubClient, apiKey: string, route: (typeof hy
return { status: "completed", result: { value: route.packageResult(blobs) } };
}
-async function synthesizeAudio(client: HypiHubClient, context: EndpointInvocationContext): Promise<EndpointFulfillment> {
+async function synthesizeAudio(client: HypiHubClient, context: EndpointInvocationContext, publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"]): Promise<EndpointFulfillment> {
const route = hypiHubRouteForCapability(context.need.capability);
assert(route !== undefined && route.media === "audio", "HypiHub does not implement this exact capability");
const compiled = await route.compile(context.need.constraints,
- (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));
+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact, publicAssetUrl, true));
+ await verifyModelRoute(client, credential(context), compiled.model, "audio_speech");
const audio = await client.binary("/audio/speech", credential(context), { model: compiled.model, ...(compiled.input as Record<string, unknown>) });
const artifact = await context.artifacts.put(audio.bytes, audio.mediaType);
return { value: route.packageResult([artifact]) };
}
-function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs: number): AsyncEndpoint {
+function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs: number, publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"]): AsyncEndpoint {
return {
async start(context: EndpointStartContext) {
try {
const route = hypiHubRouteForCapability(context.need.capability);
assert(route !== undefined, "HypiHub does not implement this exact capability"); const apiKey = credential(context);
const compiled = await route.compile(context.need.constraints,
- (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));
+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact, publicAssetUrl, false));
assert(route.media !== "audio", "HypiHub audio capabilities use an immediate endpoint");
const input = compiled.input as Record<string, unknown>;
const hasReferences = Object.entries(input).some(([key, value]) => {
@@ -145,8 +161,8 @@ function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs:
export function createHypiHubProvider(options: CreateHypiHubProviderOptions = {}) {
const client = new HypiHubClient({ baseUrl: options.baseUrl ?? "https://hypit.ai/v1", timeout: options.requestTimeoutMs ?? 30_000, fetcher: options.fetch ?? globalThis.fetch });
- const asyncEndpoint = endpoint(client, options.pollIntervalMs ?? 5_000, 20 * 60_000);
- const audioEndpoint: ImmediateEndpointHandler = (context) => synthesizeAudio(client, context);
+ const asyncEndpoint = endpoint(client, options.pollIntervalMs ?? 5_000, 20 * 60_000, options.publicAssetUrl);
+ const audioEndpoint: ImmediateEndpointHandler = (context) => synthesizeAudio(client, context, options.publicAssetUrl);
return defineEndpointPackage({
module: hypiHubProviderModuleRef, facet: "gateway", instance: options.instance ?? "hypihub.default", pool: options.pool ?? options.instance ?? "hypihub.default",
credentials: { apiKey: options.apiKey ?? credentialRef("env", "HYPIHUB_API_KEY") }, credentialInputs: { apiKey: { label: "HypiHub API key" } }, defaultConcurrency: options.defaultConcurrency ?? 4,
diff --git a/packages/provider-hypihub/src/routes.ts b/packages/provider-hypihub/src/routes.ts
index 0f9430d3..cd0aee5d 100644
--- a/packages/provider-hypihub/src/routes.ts
+++ b/packages/provider-hypihub/src/routes.ts
@@ -52,6 +52,10 @@ function normalizeHypiHubRequest(
return { model: request.model, input: canonicalize(input) };
}
+ // Audio mappings already use HypiHub's public /audio/speech vocabulary.
+ // They must not fall through the video reference normalizer below.
+ if (mapping.result === "audio") return { model: request.model, input: canonicalize(input) };
+
if (typeof input.resolution === "string") input.resolution = input.resolution.toLowerCase();
const imageRefs = input.reference_image_urls;
@@ -61,21 +65,24 @@ function normalizeHypiHubRequest(
const lastFrame = input.last_image_url;
const audioRefs = input.reference_audios;
const sourceTaskId = input.source_task_id;
- if (Array.isArray(audioRefs) && audioRefs.length > 0) {
- throw new Error("HypiHub unified video API does not support reference audio inputs");
- }
- if (Array.isArray(videoRefs) && videoRefs.length > 0) {
- throw new Error("HypiHub video reference inputs require a public HTTPS URL; the Runtime ArtifactStore cannot expose one");
- }
- if (Array.isArray(imageRefs) && imageRefs.length > 1) {
- throw new Error("HypiHub unified video API accepts one input_reference image; multiple reference images are not supported");
- }
- if (Array.isArray(genericImageRefs) && genericImageRefs.length > 1) {
- throw new Error("HypiHub unified video API accepts one input_reference image; multiple reference images are not supported");
- }
if (typeof sourceTaskId === "string" && sourceTaskId.length > 0) {
throw new Error("HypiHub unified video API does not support video continuation sourceTaskId");
}
+
+ // HypiHub's upstream relay exposes vendor-specific reference arrays through
+ // its documented `extra` passthrough. Unknown top-level fields are ignored
+ // by the public JSON decoder, while `extra` is handed to the selected
+ // upstream adaptor and becomes its provider-neutral reference vocabulary.
+ const extra = input.extra !== null && typeof input.extra === "object" && !Array.isArray(input.extra)
+ ? { ...(input.extra as Record<string, unknown>) } : {};
+ if (Array.isArray(imageRefs) && imageRefs.length > 0) extra.reference_image_urls = imageRefs;
+ if (Array.isArray(genericImageRefs) && genericImageRefs.length > 0) {
+ const urls = genericImageRefs.map((item) => item !== null && typeof item === "object"
+ ? (item as Record<string, unknown>).url : item).filter((item): item is string => typeof item === "string" && item.length > 0);
+ if (urls.length > 0) extra.reference_image_urls = urls;
+ }
+ if (Array.isArray(videoRefs) && videoRefs.length > 0) extra.reference_videos = videoRefs;
+ if (Array.isArray(audioRefs) && audioRefs.length > 0) extra.reference_audios = audioRefs;
delete input.reference_image_urls;
delete input.reference_images;
delete input.reference_videos;
@@ -83,15 +90,10 @@ function normalizeHypiHubRequest(
delete input.first_image_url;
delete input.last_image_url;
delete input.source_task_id;
+ delete input.extra;
if (typeof firstFrame === "string" && firstFrame.length > 0) input.input_reference = firstFrame;
- else if (Array.isArray(imageRefs) && imageRefs.length > 0) input.input_reference = imageRefs[0];
- else if (Array.isArray(genericImageRefs) && genericImageRefs.length > 0) {
- const first = genericImageRefs[0];
- if (first !== null && typeof first === "object" && typeof (first as Record<string, unknown>).url === "string") {
- input.input_reference = (first as Record<string, unknown>).url;
- }
- }
if (typeof lastFrame === "string" && lastFrame.length > 0) input.last_frame = lastFrame;
+ if (Object.keys(extra).length > 0) input.extra = extra;
return { model: request.model, input: canonicalize(input) };
}
diff --git a/packages/provider-hypihub/test/routes.test.ts b/packages/provider-hypihub/test/routes.test.ts
index 90392cb1..4c14fb6e 100644
--- a/packages/provider-hypihub/test/routes.test.ts
+++ b/packages/provider-hypihub/test/routes.test.ts
@@ -11,6 +11,7 @@ const image = {
};
const resolve = async () => "data:image/png;base64,AQID";
+const resolveAudio = async () => "data:audio/wav;base64,AQID";
test("HypiHub GPT image requests use canonical edit references and size dimensions", async () => {
const route = hypiHubRoutes.find((item) => item.capability.name === "gpt-image-2");
@@ -28,7 +29,7 @@ test("HypiHub GPT image requests use canonical edit references and size dimensio
prompt: "edit",
aspect_ratio: "1:1",
size: "1024x1024",
- reference_images: [{ url: "data:image/png;base64,AQID" }],
+ reference_images: ["data:image/png;base64,AQID"],
});
});
@@ -46,18 +47,77 @@ test("HypiHub image-to-video requests use input_reference", async () => {
assert.deepEqual(result.input, {
prompt: "animate",
seconds: 5,
+ resolution: "2k",
input_reference: "data:image/png;base64,AQID",
});
});
-test("HypiHub rejects reference video/audio that cannot be represented by its public API", async () => {
+test("HypiHub preserves video/audio reference arrays for its upstream passthrough", async () => {
const route = hypiHubRoutes.find((item) => item.capability.name === "minimax-h3");
assert.ok(route);
- await assert.rejects(() => route.compile({
+ const result = await route.compile({
ports: {
prompt: ["animate"],
duration: [5],
referenceAudio: [{ role: "audio", artifact: { ...image, mediaType: "audio/wav" } }],
},
- }, resolve), /does not support reference audio/);
+ }, resolveAudio);
+ assert.deepEqual(result.input, {
+ prompt: "animate", seconds: 5, resolution: "2k",
+ extra: { reference_audios: ["data:audio/wav;base64,AQID"] },
+ });
+});
+
+test("HypiHub Seedance multimodal references ride in the public extra passthrough", async () => {
+ const route = hypiHubRoutes.find((item) => item.capability.name === "seedance-2");
+ assert.ok(route);
+ const result = await route.compile({
+ ports: {
+ prompt: ["animate"],
+ duration: [5],
+ resolution: ["720p"],
+ aspectRatio: ["16:9"],
+ generateAudio: [true],
+ webSearch: [false],
+ referenceImage: [
+ { role: "image", artifact: image },
+ { role: "image", artifact: { ...image, digest: `sha256:${"2".repeat(64)}` } },
+ ],
+ },
+ }, resolve);
+ assert.equal(result.model, "bytedance/seedance-2");
+ assert.deepEqual(result.input, {
+ prompt: "animate",
+ seconds: 5,
+ resolution: "720p",
+ aspect_ratio: "16:9",
+ generate_audio: true,
+ web_search: false,
+ extra: { reference_image_urls: ["data:image/png;base64,AQID", "data:image/png;base64,AQID"] },
+ });
+});
+
+test("HypiHub MiniMax reference mode preserves image arrays in extra", async () => {
+ const route = hypiHubRoutes.find((item) => item.capability.name === "minimax-h3");
+ assert.ok(route);
+ const result = await route.compile({ ports: {
+ prompt: ["animate"], duration: [6], aspectRatio: ["16:9"],
+ referenceImage: [{ role: "image", artifact: image }],
+ } }, resolve);
+ assert.equal(result.model, "minimax-h3/reference-to-video");
+ assert.deepEqual(result.input, {
+ prompt: "animate", seconds: 6, aspect_ratio: "16:9",
+ resolution: "2k",
+ extra: { reference_image_urls: ["data:image/png;base64,AQID"] },
+ });
+});
+
+test("HypiHub MiMo TTS mappings use the public audio speech fields", async () => {
+ const voiceDesign = hypiHubRoutes.find((item) => item.capability.name === "mimo-v2.5-tts-voicedesign");
+ assert.ok(voiceDesign);
+ const result = await voiceDesign.compile({
+ ports: { text: ["hello"], voiceDescription: ["warm and calm"] },
+ }, resolve);
+ assert.equal(result.model, "mimo-v2.5-tts-voicedesign");
+ assert.deepEqual(result.input, { input: "hello", voice_description: "warm and calm" });
});