Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(provider-hypihub): upload all reference media through files
rpone401f4372dbfa
diff --git a/.gitignore b/.gitignoreindex 464bab39..6fd3e12f 100644--- a/.gitignore+++ b/.gitignore@@ -67,4 +67,6 @@ examples/**/.svml//examples/reference-reconstruction/# Reverse-engineering scratch projects live outside the repository too-/projects/+/projects/*+!/projects/hypihub-provider-smoke/+!/projects/hypihub-provider-smoke/**diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.mdindex f481ca37..c441a7ba 100644--- a/packages/provider-hypihub/README.md+++ b/packages/provider-hypihub/README.md@@ -7,8 +7,9 @@ when its key is configured.It maps the currently shipped image/video model capabilities to HypiHub, including image edits andimage-to-video first-frame inputs, submits jobs, polls them, downloads the first-class assets andpersists them in Hypit's configured ArtifactStore. Image references use HypiHub's documented-`reference_images` object shape (`[{ "url": "…" }]`); video reference arrays use the documented `extra` passthrough so HypiHub can hand them to its-selected upstream adaptor. First/last-frame images use `input_reference` and `last_frame`. It also+`reference_images` object shape (`[{ "url": "…" }]`); video references use the public+`reference_image_urls`, `reference_videos`, and `reference_audios` fields (with `ref_video_url`+for one video). First/last-frame images use `first_frame` and `last_frame`. It alsoexports a small Gemini-native VLM generator for callers that previously used Vertex.Runtime Profile example:@@ -37,3 +38,6 @@ credentials should be resolved at [hypit.ai](https://hypit.ai). Referenced imageArtifacts are uploaded automatically through `POST /v1/files`, then their returned HTTPS capabilityURLs are used in image and video requests. Uploads are deduplicated by Artifact digest within oneRuntime operation. Embedded callers may override that transport with `publicAssetUrl`.++Set `audio: false` when official `@hypit/provider-xiaomi-mimo` should own MiMo audio capabilities;+one Runtime can then use HypiHub for image/video and the official MiMo endpoint for audio.diff --git a/packages/provider-hypihub/src/activation.ts b/packages/provider-hypihub/src/activation.tsindex a9a78ca1..c1daa94a 100644--- a/packages/provider-hypihub/src/activation.ts+++ b/packages/provider-hypihub/src/activation.ts@@ -2,6 +2,7 @@ import {createRuntimeEndpointAdapterFacet,runtimeConfigCredentialRef,runtimeConfigExact,+ runtimeConfigBoolean,runtimeConfigObject,runtimeConfigPositiveInteger,runtimeConfigString,@@ -14,7 +15,7 @@ const adapter = createRuntimeEndpointAdapterFacet({activate(context) {if (context.pool === undefined) throw new Error("HypiHub Provider Pool is required");const config = runtimeConfigObject(context.config, "HypiHub");- runtimeConfigExact(config, ["baseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs", "requestTimeoutMs"], "HypiHub");+ runtimeConfigExact(config, ["baseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs", "requestTimeoutMs", "audio"], "HypiHub");const baseUrl = runtimeConfigString(config.baseUrl, "HypiHub baseUrl");if (baseUrl !== undefined) {const url = new URL(baseUrl);@@ -27,6 +28,7 @@ const adapter = createRuntimeEndpointAdapterFacet({const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "HypiHub defaultConcurrency");const pollIntervalMs = runtimeConfigPositiveInteger(config.pollIntervalMs, "HypiHub pollIntervalMs");const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "HypiHub requestTimeoutMs");+ const audio = runtimeConfigBoolean(config.audio, "HypiHub audio");return {endpoint: createHypiHubProvider({instance: context.instance,@@ -36,6 +38,7 @@ const adapter = createRuntimeEndpointAdapterFacet({...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),...(pollIntervalMs === undefined ? {} : { pollIntervalMs }),...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),+ ...(audio === undefined ? {} : { audio }),}),};},diff --git a/packages/provider-hypihub/src/gemini.ts b/packages/provider-hypihub/src/gemini.tsindex b744f8b1..dfe67f87 100644--- a/packages/provider-hypihub/src/gemini.ts+++ b/packages/provider-hypihub/src/gemini.ts@@ -1,6 +1,7 @@export type HypiHubGeminiPart =| { readonly text: string }- | { readonly inlineData: { readonly mimeType: string; readonly data: string } };+ | { readonly inlineData: { readonly mimeType: string; readonly data: string } }+ | { readonly fileData: { readonly mimeType?: string; readonly fileUri: string } };export type HypiHubGeminiGeneratorOptions = {readonly apiKey: string;@@ -19,14 +20,26 @@ function assert(condition: unknown, message: string): asserts condition {if (!condition) throw new Error(message);}-function partValue(part: HypiHubGeminiPart): Record<string, unknown> {- if ("text" in part) return { text: part.text };+function partValue(part: HypiHubGeminiPart, uploaded: Map<string, Promise<string>>,+ upload: (mimeType: string, data: string) => Promise<string>): Promise<Record<string, unknown>> {+ if ("text" in part) return Promise.resolve({ text: part.text });+ if ("fileData" in part) {+ assert(typeof part.fileData.fileUri === "string" && part.fileData.fileUri.length > 0,+ "HypiHub Gemini file media part is incomplete");+ return Promise.resolve({ fileData: { ...(part.fileData.mimeType === undefined ? {} : { mimeType: part.fileData.mimeType }), fileUri: part.fileData.fileUri } });+ }if ("inlineData" in part) {assert(typeof part.inlineData.mimeType === "string" && typeof part.inlineData.data === "string","HypiHub Gemini inline media part is incomplete");- return { inlineData: { mimeType: part.inlineData.mimeType, data: part.inlineData.data } };+ const key = `${part.inlineData.mimeType}\0${part.inlineData.data}`;+ let pending = uploaded.get(key);+ if (pending === undefined) {+ pending = upload(part.inlineData.mimeType, part.inlineData.data);+ uploaded.set(key, pending);+ }+ return pending.then((fileUri) => ({ fileData: { mimeType: part.inlineData.mimeType, fileUri } }));}- throw new Error("HypiHub Gemini only supports text and inline media parts");+ throw new Error("HypiHub Gemini only supports text, inline media and file media parts");}/** Create a small native-Gemini generator for callers that previously used Vertex. */@@ -47,13 +60,37 @@ export function createHypiHubGeminiGenerator(options: HypiHubGeminiGeneratorOpticonst controller = new AbortController();const timer = setTimeout(() => controller.abort(), timeout);try {+ const uploaded = new Map<string, Promise<string>>();+ const upload = async (mimeType: string, encoded: string): Promise<string> => {+ let bytes: Uint8Array;+ try { bytes = new Uint8Array(Buffer.from(encoded, "base64")); }+ catch (error) { throw new Error("HypiHub Gemini inline media is not valid base64", { cause: error }); }+ assert(bytes.byteLength > 0, "HypiHub Gemini inline media is empty");+ const form = new FormData();+ const copy = new ArrayBuffer(bytes.byteLength);+ new Uint8Array(copy).set(bytes);+ form.append("file", new Blob([copy], { type: mimeType }), `gemini-reference.${extension(mimeType)}`);+ form.append("purpose", "reference");+ const uploadedResponse = await fetcher(`${baseUrl}/v1/files`, {+ method: "POST", signal: controller.signal,+ headers: { authorization: `Bearer ${apiKey}` }, body: form,+ });+ const uploadedText = await uploadedResponse.text();+ if (!uploadedResponse.ok) throw new Error(`HypiHub Gemini file upload returned HTTP ${uploadedResponse.status}: ${uploadedText.slice(0, 300)}`);+ let uploadedBody: unknown;+ try { uploadedBody = JSON.parse(uploadedText); } catch { throw new Error("HypiHub Gemini file upload returned invalid JSON"); }+ const url = (uploadedBody as { readonly url?: unknown }).url;+ assert(typeof url === "string" && /^https:\/\//iu.test(url), "HypiHub Gemini file upload returned no HTTPS URL");+ return url;+ };+ const parts = await Promise.all(input.parts.map((part) => partValue(part, uploaded, upload)));const response = await fetcher(`${baseUrl}/v1beta/models/${encodeURIComponent(model)}:generateContent`, {method: "POST",signal: controller.signal,headers: { "x-goog-api-key": apiKey, "content-type": "application/json" },body: JSON.stringify({systemInstruction: { parts: [{ text: input.instruction }] },- contents: [{ role: "user", parts: input.parts.map(partValue) }],+ contents: [{ role: "user", parts }],generationConfig: { temperature: 1, responseMimeType: "text/plain" },}),});@@ -78,3 +115,12 @@ export function createHypiHubGeminiGenerator(options: HypiHubGeminiGeneratorOpti}};}++function extension(mimeType: string): string {+ const value: Readonly<Record<string, string>> = {+ "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp",+ "audio/mpeg": "mp3", "audio/wav": "wav", "audio/mp4": "m4a",+ "video/mp4": "mp4", "video/quicktime": "mov", "video/webm": "webm",+ };+ return value[mimeType.toLowerCase()] ?? "bin";+}diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.tsindex 2b3327e6..cef359af 100644--- a/packages/provider-hypihub/src/mapping.ts+++ b/packages/provider-hypihub/src/mapping.ts@@ -17,8 +17,8 @@ const seedance = (name: string, model: string): GenerationWireMapping => ({referenceImage: { as: "urlArray", field: "reference_image_urls" },referenceVideo: { as: "urlArray", field: "reference_videos" },referenceAudio: { as: "urlArray", field: "reference_audios" },- firstFrame: { as: "url", field: "first_image_url" },- lastFrame: { as: "url", field: "last_image_url" },+ firstFrame: { as: "url", field: "first_frame" },+ lastFrame: { as: "url", field: "last_frame" },resolution: { as: "value", field: "resolution" },aspectRatio: { as: "value", field: "aspect_ratio" },duration: { as: "value", field: "seconds" },@@ -81,8 +81,8 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [referenceImage: { as: "urlArray", field: "reference_image_urls" },referenceVideo: { as: "urlArray", field: "reference_videos" },referenceAudio: { as: "urlArray", field: "reference_audios" },- firstFrame: { as: "url", field: "first_image_url" },- lastFrame: { as: "url", field: "last_image_url" },+ firstFrame: { as: "url", field: "first_frame" },+ lastFrame: { as: "url", field: "last_frame" },},},{diff --git a/packages/provider-hypihub/src/provider.ts b/packages/provider-hypihub/src/provider.tsindex ee23e93f..34a274f8 100644--- a/packages/provider-hypihub/src/provider.ts+++ b/packages/provider-hypihub/src/provider.ts@@ -16,6 +16,8 @@ export type CreateHypiHubProviderOptions = {readonly defaultConcurrency?: number;readonly pollIntervalMs?: number;readonly requestTimeoutMs?: number;+ /** Expose HypiHub's MiMo audio capabilities. Disable when official MiMo owns audio. */+ readonly audio?: boolean;readonly fetch?: typeof globalThis.fetch;/** Overrides the default POST /v1/files upload for referenced artifacts. */readonly publicAssetUrl?: (artifact: BlobRef, artifacts: ArtifactStore) => Promise<string>;@@ -174,7 +176,7 @@ function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs:assert(route.media !== "audio", "HypiHub audio capabilities use an immediate endpoint");const input = compiled.input as Record<string, unknown>;const hasReferences = Object.entries(input).some(([key, value]) => {- if (!["images", "reference_images", "reference_image_urls", "reference_videos", "reference_audios", "first_image_url", "last_image_url"].includes(key)) return false;+ if (!["images", "reference_images", "reference_image_urls", "reference_videos", "reference_audios", "first_frame", "last_frame"].includes(key)) return false;return Array.isArray(value) ? value.length > 0 : typeof value === "string" && value.length > 0;});const path = route.media === "image"@@ -220,8 +222,10 @@ export function createHypiHubProvider(options: CreateHypiHubProviderOptions = {}return defineEndpointPackage({module: hypiHubProviderModuleRef, facet: "gateway", instance: options.instance ?? "hypihub.default", pool: options.pool ?? options.instance ?? "hypihub.default",credentials: { apiKey: options.apiKey ?? credentialRef("env", "HYPIHUB_API_KEY") }, credentialInputs: { apiKey: { label: "HypiHub API key" } }, defaultConcurrency: options.defaultConcurrency ?? 4,- capabilities: hypiHubRoutes.map((route) => route.media === "audio"- ? { capability: route.capability, returns: route.returns, lifecycle: "immediate" as const, handler: audioEndpoint, lane: route.capability.name }- : { capability: route.capability, returns: route.returns, lifecycle: "asynchronous" as const, endpoint: asyncEndpoint, lane: route.capability.name }),+ capabilities: hypiHubRoutes+ .filter((route) => options.audio !== false || route.media !== "audio")+ .map((route) => route.media === "audio"+ ? { capability: route.capability, returns: route.returns, lifecycle: "immediate" as const, handler: audioEndpoint, lane: route.capability.name }+ : { capability: route.capability, returns: route.returns, lifecycle: "asynchronous" as const, endpoint: asyncEndpoint, lane: route.capability.name }),});}diff --git a/packages/provider-hypihub/src/routes.ts b/packages/provider-hypihub/src/routes.tsindex 0bc281bd..745f62ec 100644--- a/packages/provider-hypihub/src/routes.ts+++ b/packages/provider-hypihub/src/routes.ts@@ -70,8 +70,8 @@ function normalizeHypiHubRequest(const imageRefs = input.reference_image_urls;const genericImageRefs = input.reference_images;const videoRefs = input.reference_videos;- const firstFrame = input.first_image_url;- const lastFrame = input.last_image_url;+ const firstFrame = input.first_frame;+ const lastFrame = input.last_frame;const audioRefs = input.reference_audios;const sourceTaskId = input.source_task_id;if (typeof sourceTaskId === "string" && sourceTaskId.length > 0) {@@ -85,8 +85,8 @@ function normalizeHypiHubRequest(delete input.reference_images;delete input.reference_videos;delete input.reference_audios;- delete input.first_image_url;- delete input.last_image_url;+ delete input.first_frame;+ delete input.last_frame;delete input.source_task_id;if (typeof firstFrame === "string" && firstFrame.length > 0) input.first_frame = firstFrame;if (typeof lastFrame === "string" && lastFrame.length > 0) input.last_frame = lastFrame;diff --git a/packages/provider-hypihub/test/gemini.test.ts b/packages/provider-hypihub/test/gemini.test.tsindex 9643787d..c1d07e7a 100644--- a/packages/provider-hypihub/test/gemini.test.ts+++ b/packages/provider-hypihub/test/gemini.test.ts@@ -6,12 +6,19 @@ import { createHypiHubGeminiGenerator } from "../src/gemini.js";test("HypiHub Gemini uses native wire format for text, image and video parts", async () => {let seenUrl = "";let seenBody: Record<string, unknown> | undefined;+ let uploads = 0;const generate = createHypiHubGeminiGenerator({apiKey: "test-key",model: "gemini-3.7-flash",baseUrl: "https://hypit.ai/v1",fetch: async (input, init) => {seenUrl = String(input);+ if (seenUrl.endsWith("/v1/files")) {+ uploads += 1;+ assert.equal(init?.method, "POST");+ assert.ok(init?.body instanceof FormData);+ return Response.json({ url: `https://hypit.ai/files/ref-${uploads}.png` });+ }seenBody = JSON.parse(String(init?.body)) as Record<string, unknown>;return Response.json({ candidates: [{ content: { parts: [{ text: "OK" }] } }] });},@@ -23,10 +30,11 @@ test("HypiHub Gemini uses native wire format for text, image and video parts", a{ inlineData: { mimeType: "video/mp4", data: "dmlkZW8=" } },] }), "OK");assert.equal(seenUrl, "https://hypit.ai/v1beta/models/gemini-3.7-flash:generateContent");+ assert.equal(uploads, 2);assert.deepEqual(seenBody?.contents, [{ role: "user", parts: [{ text: "inspect" },- { inlineData: { mimeType: "image/png", data: "aW1hZ2U=" } },- { inlineData: { mimeType: "video/mp4", data: "dmlkZW8=" } },+ { fileData: { mimeType: "image/png", fileUri: "https://hypit.ai/files/ref-1.png" } },+ { fileData: { mimeType: "video/mp4", fileUri: "https://hypit.ai/files/ref-2.png" } },] }]);});diff --git a/projects/hypihub-provider-smoke/assets/reference.wav b/projects/hypihub-provider-smoke/assets/reference.wavnew file mode 100644index 00000000..72d16c0dBinary files /dev/null and b/projects/hypihub-provider-smoke/assets/reference.wav differdiff --git a/projects/hypihub-provider-smoke/audio.svrun b/projects/hypihub-provider-smoke/audio.svrunnew file mode 100644index 00000000..6b1d955b--- /dev/null+++ b/projects/hypihub-provider-smoke/audio.svrun@@ -0,0 +1,6 @@+<?svml using="@hypit/run-markup@1"?>++<svrun version="1">+ <author source="./main.svml"/>+ <target output="mimo-voice.audio"/>+</svrun>diff --git a/projects/hypihub-provider-smoke/build.svrun b/projects/hypihub-provider-smoke/build.svrunnew file mode 100644index 00000000..7d905ca0--- /dev/null+++ b/projects/hypihub-provider-smoke/build.svrun@@ -0,0 +1,18 @@+<?svml using="@hypit/run-markup@1"?>++<svrun version="1">+ <author source="./main.svml"/>+ <target output="gpt-text.image"/>+ <target output="gpt-edit.image"/>+ <target output="nano-text.image"/>+ <target output="nano-edit.image"/>+ <target output="seedream-text.image"/>+ <target output="seedream-edit.image"/>+ <target output="seedance-text.video"/>+ <target output="seedance-image-audio.video"/>+ <target output="minimax-text.video"/>+ <target output="minimax-reference.video"/>+ <target output="grok-text.video"/>+ <target output="grok-image.video"/>+ <target output="mimo-voice.audio"/>+</svrun>diff --git a/projects/hypihub-provider-smoke/hypit.runtime.json b/projects/hypihub-provider-smoke/hypit.runtime.jsonnew file mode 100644index 00000000..600d2dee--- /dev/null+++ b/projects/hypihub-provider-smoke/hypit.runtime.json@@ -0,0 +1,35 @@+{+ "format": "hypit.runtime-profile@1",+ "runtime": {+ "use": "@hypit/runtime-local",+ "config": {+ "dataRoot": ".hypit/runtimes/local",+ "artifacts": { "use": "@hypit/artifact-store-fs", "config": { "path": "artifacts" } },+ "credentials": { "env": { "use": "@hypit/credential-store-env" } },+ "endpoints": {+ "hypihub.smoke": {+ "use": "@hypit/provider-hypihub",+ "pool": "hypihub.smoke",+ "config": {+ "baseUrl": "https://hypit.ai/v1",+ "apiKey": { "store": "env", "key": "HYPIHUB_API_KEY" },+ "defaultConcurrency": 1,+ "pollIntervalMs": 5000,+ "requestTimeoutMs": 120000,+ "audio": false+ }+ },+ "xiaomi-mimo.smoke": {+ "use": "@hypit/provider-xiaomi-mimo",+ "pool": "xiaomi-mimo.smoke",+ "config": {+ "apiBaseUrl": "https://api.xiaomimimo.com/v1",+ "apiKey": { "store": "env", "key": "MIMO_API_KEY" },+ "defaultConcurrency": 2,+ "requestTimeoutMs": 180000+ }+ }+ }+ }+ }+}diff --git a/projects/hypihub-provider-smoke/main.svml b/projects/hypihub-provider-smoke/main.svmlnew file mode 100644index 00000000..acb1a95d--- /dev/null+++ b/projects/hypihub-provider-smoke/main.svml@@ -0,0 +1,53 @@+<?svml using="@hypit/markup@1"?>++<svml>+ <import as="asset" from="@hypit/media@1"/>+ <import as="text" from="@hypit/text@1"/>+ <import as="gpt" from="@hypit/gpt-image@1"/>+ <import as="nano" from="@hypit/nano-banana@1"/>+ <import as="seedream" from="@hypit/seedream@1"/>+ <import as="seedance" from="@hypit/seedance@1"/>+ <import as="h3" from="@hypit/minimax-h3@1"/>+ <import as="grok" from="@hypit/grok-imagine@1"/>+ <import as="mimo" from="@hypit/mimo-tts@1"/>++ <text:Value id="prompt">A tiny red paper boat on a blue studio table, clean product photograph, no text.</text:Value>+ <text:Value id="edit-prompt">Keep the paper boat unchanged and make the studio background warm orange, no text.</text:Value>+ <text:Value id="motion-prompt">The paper boat glides slowly across the blue table in one continuous locked-camera shot.</text:Value>+ <text:Value id="speech">The red paper boat moves across the blue table.</text:Value>+ <asset:Image id="reference-image" src="./assets/reference.png"/>+ <asset:Audio id="reference-audio" src="./assets/reference.wav"/>++ <gpt:Image id="gpt-text" prompt={prompt} aspect-ratio="1:1" resolution="1K"/>+ <gpt:Image id="gpt-edit" prompt={edit-prompt} aspect-ratio="1:1" resolution="1K">+ <gpt:Reference image={reference-image}/>+ </gpt:Image>++ <nano:ProImage id="nano-text" prompt={prompt} aspect-ratio="1:1" resolution="1K" output-format="png"/>+ <nano:ProImage id="nano-edit" prompt={edit-prompt} aspect-ratio="1:1" resolution="1K" output-format="png">+ <nano:Reference image={reference-image}/>+ </nano:ProImage>++ <seedream:TextImage id="seedream-text" prompt={prompt} aspect-ratio="1:1" quality="basic" output-format="png" nsfw-check="true"/>+ <seedream:ReferenceImage id="seedream-edit" prompt={edit-prompt} aspect-ratio="1:1" quality="basic" output-format="png" nsfw-check="true">+ <seedream:Reference image={reference-image}/>+ </seedream:ReferenceImage>++ <seedance:TextVideo id="seedance-text" model="mini" prompt={motion-prompt} duration="5" resolution="480p" aspect-ratio="16:9" generate-audio="false"/>+ <seedance:ReferenceVideo id="seedance-image-audio" model="mini" prompt={motion-prompt} duration="5" resolution="480p" aspect-ratio="16:9" generate-audio="true">+ <seedance:Reference image={reference-image}/>+ <seedance:Reference audio={reference-audio}/>+ </seedance:ReferenceVideo>++ <h3:TextVideo id="minimax-text" prompt={motion-prompt} duration="6" resolution="768P" aspect-ratio="16:9"/>+ <h3:ReferenceVideo id="minimax-reference" prompt={motion-prompt} duration="6" resolution="768P" aspect-ratio="16:9">+ <h3:Reference image={reference-image}/>+ </h3:ReferenceVideo>++ <grok:PreviewVideo id="grok-text" prompt={motion-prompt} duration="6" aspect-ratio="16:9" resolution="480p"/>+ <grok:Video id="grok-image" prompt={motion-prompt} duration="6" aspect-ratio="16:9" resolution="480p">+ <grok:Reference image={reference-image}/>+ </grok:Video>++ <mimo:VoiceDesign id="mimo-voice" speech={speech}>A clear, warm narrator with a lightly playful delivery.</mimo:VoiceDesign>+</svml>diff --git a/projects/hypihub-provider-smoke/package.json b/projects/hypihub-provider-smoke/package.jsonnew file mode 100644index 00000000..c7548a28--- /dev/null+++ b/projects/hypihub-provider-smoke/package.json@@ -0,0 +1,6 @@+{+ "name": "hypihub-provider-smoke",+ "version": "0.0.0",+ "private": true,+ "type": "module"+}