Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat: align HypiHub references and extract Vertex provider
rpone360b13770035
diff --git a/package.json b/package.jsonindex d9728007..00639431 100644--- a/package.json+++ b/package.json@@ -100,6 +100,7 @@"@hypit/provider-image-opencv-local": "workspace:*","@hypit/provider-hypihub": "workspace:*","@hypit/provider-kie": "workspace:*",+ "@hypit/provider-vertex": "workspace:*","@hypit/provider-media-aws-lambda": "workspace:*","@hypit/provider-media-local": "workspace:*","@hypit/preview-mock": "workspace:*",diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.mdindex d395afe5..50c7349e 100644--- a/packages/provider-hypihub/README.md+++ b/packages/provider-hypihub/README.md@@ -4,11 +4,11 @@ Thin Hypit Runtime Provider for a HypiHub `/v1` deployment. It is an optional depaid generation and Gemini VLM requests; callers may keep their own Provider and select HypiHub onlywhen its key is configured.-It maps the currently shipped image/video model capabilities to HypiHub, submits jobs, polls them,-downloads the first-class assets and persists them in Hypit's configured ArtifactStore. It also-exports a small Gemini-native VLM generator for callers that previously used Vertex. The generation-adapter intentionally supports prompt-only requests; reference-media inputs are rejected until a-shared Artifact URL bridge is needed.+It maps the currently shipped image/video model capabilities to HypiHub, including image edits and+image/video reference inputs, submits jobs, polls them, downloads the first-class assets and persists+them in Hypit's configured ArtifactStore. References are sent as HypiHub-compatible `data:` URLs, so+the adapter does not require a provider-specific public upload bucket. It also exports a small+Gemini-native VLM generator for callers that previously used Vertex.Runtime Profile example:diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.tsindex f82c5af7..7d3ba952 100644--- a/packages/provider-hypihub/src/mapping.ts+++ b/packages/provider-hypihub/src/mapping.ts@@ -14,9 +14,16 @@ const seedance = (name: string, model: string): GenerationWireMapping => ({capability: { module: SEEDANCE, name }, result: "video", routes: [{ model }],fields: {prompt: { as: "value", field: "prompt" },+ referenceImage: { as: "urlArray", field: "reference_image_urls" },+ referenceVideo: { as: "urlArray", field: "reference_videos" },+ referenceAudio: { as: "urlArray", field: "reference_audios" },+ firstFrame: { as: "url", field: "first_image_url" },+ lastFrame: { as: "url", field: "last_image_url" },resolution: { as: "value", field: "resolution" },aspectRatio: { as: "value", field: "aspect_ratio" },duration: { as: "value", field: "seconds" },+ generateAudio: { as: "value", field: "generate_audio" },+ webSearch: { as: "value", field: "web_search" },},});@@ -26,11 +33,15 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [seedance("seedance-2-mini", "bytedance/seedance-2-mini"),seedance("seedance-2.5", "bytedance/seedance-2-5"),{- capability: { module: GPT_IMAGE, name: "gpt-image-2" }, result: "image", routes: [{ model: "gpt-image-2-text-to-image" }],+ capability: { module: GPT_IMAGE, name: "gpt-image-2" }, result: "image", routes: [+ { model: "gpt-image-2-image-to-image", whenPresent: ["images"] },+ { model: "gpt-image-2-text-to-image" },+ ],fields: {prompt: { as: "value", field: "prompt" },aspectRatio: { as: "value", field: "aspect_ratio" },resolution: { as: "value", field: "size", whenAbsent: "1024x1024" },+ images: { as: "urlArray", field: "images" },},},...(["nano-banana-2", "nano-banana-pro"] as const).map((model) => ({@@ -39,35 +50,58 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [prompt: { as: "value" as const, field: "prompt" },aspectRatio: { as: "value" as const, field: "aspect_ratio" },resolution: { as: "value" as const, field: "size", whenAbsent: "1024x1024" },+ images: { as: "itemObject" as const, field: "reference_images", urlKey: "url", fieldKeys: {} },},})),{- capability: { module: SEEDREAM, name: "seedream-5-lite" }, result: "image", routes: [{ model: "seedream/5-lite-text-to-image" }],+ capability: { module: SEEDREAM, name: "seedream-5-lite" }, result: "image", routes: [+ { model: "seedream/5-pro-image-to-image", whenPresent: ["images"] },+ { model: "seedream/5-lite-text-to-image" },+ ],fields: {prompt: { as: "value", field: "prompt" },aspectRatio: { as: "value", field: "aspect_ratio" },resolution: { as: "value", field: "size", whenAbsent: "2048x2048" },+ images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },},},{- capability: { module: MINIMAX, name: "minimax-h3" }, result: "video", routes: [{ model: "minimax-h3/text-to-video" }],+ capability: { module: MINIMAX, name: "minimax-h3" }, result: "video", routes: [+ { model: "minimax-h3/image-to-video", whenPresent: ["firstFrame"] },+ { model: "minimax-h3/reference-to-video", whenPresent: ["referenceImage"] },+ { model: "minimax-h3/reference-to-video", whenPresent: ["referenceVideo"] },+ { model: "minimax-h3/text-to-video" },+ ],fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },+ referenceImage: { as: "urlArray", field: "reference_image_urls" },+ referenceVideo: { as: "urlArray", field: "reference_videos" },+ referenceAudio: { as: "urlArray", field: "reference_audios" },+ firstFrame: { as: "url", field: "first_image_url" },+ lastFrame: { as: "url", field: "last_image_url" },},},{- capability: { module: GROK, name: "grok-imagine-video" }, result: "video", routes: [{ model: "grok-imagine/text-to-video" }],+ capability: { module: GROK, name: "grok-imagine-video" }, result: "video", routes: [+ { model: "grok-imagine/image-to-video", whenPresent: ["images"] },+ { model: "grok-imagine/text-to-video" },+ ],fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },+ images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },},},{- capability: { module: GROK, name: "grok-imagine-video-1.5-preview" }, result: "video", routes: [{ model: "grok-imagine/text-to-video" }],+ capability: { module: GROK, name: "grok-imagine-video-1.5-preview" }, result: "video", routes: [+ { model: "grok-imagine/image-to-video", whenPresent: ["images"] },+ { model: "grok-imagine/text-to-video" },+ ],fields: {prompt: { as: "value", field: "prompt" }, duration: { as: "value", field: "seconds" },resolution: { as: "value", field: "resolution" }, aspectRatio: { as: "value", field: "aspect_ratio" },+ images: { as: "itemObject", field: "reference_images", urlKey: "url", fieldKeys: {} },},},{diff --git a/packages/provider-hypihub/src/provider.ts b/packages/provider-hypihub/src/provider.tsindex d4b90450..73ebb45a 100644--- a/packages/provider-hypihub/src/provider.ts+++ b/packages/provider-hypihub/src/provider.ts@@ -41,6 +41,16 @@ function jobId(value: Record<string, unknown>): string {return id;}+function dataUrl(bytes: Uint8Array, mediaType: string): string {+ return `data:${mediaType};base64,${Buffer.from(bytes).toString("base64")}`;+}++async function resolveArtifactAsDataUrl(artifacts: ArtifactStore, artifact: BlobRef): Promise<string> {+ const bytes = await artifacts.get(artifact.digest);+ assert(bytes !== undefined, `HypiHub reference artifact ${artifact.digest} is unavailable`);+ return dataUrl(bytes, artifact.mediaType);+}+class HypiHubClient {readonly baseUrl: string;readonly timeout: number;@@ -79,9 +89,8 @@ async function complete(client: HypiHubClient, apiKey: string, route: (typeof hyasync function synthesizeAudio(client: HypiHubClient, context: EndpointInvocationContext): Promise<EndpointFulfillment> {const route = hypiHubRouteForCapability(context.need.capability);assert(route !== undefined && route.media === "audio", "HypiHub does not implement this exact capability");- const compiled = await route.compile(context.need.constraints, async () => {- throw new Error("HypiHub reference media is not supported yet");- });+ const compiled = await route.compile(context.need.constraints,+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));const audio = await client.binary("/audio/speech", credential(context), { model: compiled.model, ...(compiled.input as Record<string, unknown>) });const artifact = await context.artifacts.put(audio.bytes, audio.mediaType);return { value: route.packageResult([artifact]) };@@ -93,9 +102,18 @@ function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs:try {const route = hypiHubRouteForCapability(context.need.capability);assert(route !== undefined, "HypiHub does not implement this exact capability"); const apiKey = credential(context);- const compiled = await route.compile(context.need.constraints, async () => { throw new Error("HypiHub reference media is not supported yet"); });+ const compiled = await route.compile(context.need.constraints,+ (artifact) => resolveArtifactAsDataUrl(context.artifacts, artifact));assert(route.media !== "audio", "HypiHub audio capabilities use an immediate endpoint");- const response = await client.json(route.media === "image" ? "/images/generations" : "/videos", apiKey, { method: "POST", headers: { "content-type": "application/json", "idempotency-key": context.operation }, body: JSON.stringify({ model: compiled.model, ...(compiled.input as Record<string, unknown>) }) });+ const input = compiled.input as Record<string, unknown>;+ const hasReferences = Object.entries(input).some(([key, value]) => {+ if (!["images", "reference_images", "reference_image_urls", "reference_videos", "reference_audios", "first_image_url", "last_image_url"].includes(key)) return false;+ return Array.isArray(value) ? value.length > 0 : typeof value === "string" && value.length > 0;+ });+ const path = route.media === "image"+ ? (hasReferences ? "/images/edits" : "/images/generations")+ : "/videos";+ const response = await client.json(path, apiKey, { method: "POST", headers: { "content-type": "application/json", "idempotency-key": context.operation }, body: JSON.stringify({ model: compiled.model, ...input }) });const status = response.status; if (status === "succeeded" || status === "completed") return await complete(client, apiKey, route, jobId(response), context.artifacts);const handle: Handle = { contract: "hypit.hypihub-operation@1", jobId: jobId(response), route: capabilityKey(route.capability), startedAt: Date.now() };return wakeAfter(canonicalize(handle), pollIntervalMs, Date.now(), { phase: "submitted" });diff --git a/packages/provider-vertex/README.md b/packages/provider-vertex/README.mdnew file mode 100644index 00000000..20a12c11--- /dev/null+++ b/packages/provider-vertex/README.md@@ -0,0 +1,15 @@+# `@hypit/provider-vertex`++Vertex Gemini backend for callers that need text, image or video inline-data input. The package owns+the Google GenAI client and credential parsing; callers only select the model and provide a generator+input.++```ts+import { createVertexGeminiGenerator } from "@hypit/provider-vertex";++const generate = createVertexGeminiGenerator({+ project: process.env.GOOGLE_CLOUD_PROJECT!,+ credentials: process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON!,+ model: "gemini-3.1-pro-preview",+});+```diff --git a/packages/provider-vertex/package.json b/packages/provider-vertex/package.jsonnew file mode 100644index 00000000..856b3673--- /dev/null+++ b/packages/provider-vertex/package.json@@ -0,0 +1,11 @@+{+ "name": "@hypit/provider-vertex",+ "version": "0.0.0-dev",+ "license": "SEE LICENSE IN LICENSE",+ "private": true,+ "type": "module",+ "exports": { ".": "./src/index.ts" },+ "dependencies": {+ "@google/genai": "1.52.0"+ }+}diff --git a/packages/provider-vertex/src/gemini.ts b/packages/provider-vertex/src/gemini.tsnew file mode 100644index 00000000..b0aade1f--- /dev/null+++ b/packages/provider-vertex/src/gemini.ts@@ -0,0 +1,58 @@+import { GoogleGenAI } from "@google/genai";+import type { Content } from "@google/genai";++export type VertexGeminiPart =+ | { readonly text?: string; readonly inlineData?: never }+ | { readonly inlineData?: { readonly mimeType: string; readonly data: string }; readonly text?: never };++export type VertexGeminiGenerateInput = {+ readonly parts: readonly VertexGeminiPart[];+ readonly instruction: string;+};++export type VertexGeminiGeneratorOptions = {+ readonly project: string;+ readonly credentials: Record<string, unknown> | string;+ readonly location?: string;+ readonly model?: string;+ readonly requestTimeoutMs?: number;+};++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++function parseCredentials(value: Record<string, unknown> | string): Record<string, unknown> {+ if (typeof value !== "string") return value;+ let parsed: unknown;+ try { parsed = JSON.parse(value); } catch { throw new Error("GOOGLE_APPLICATION_CREDENTIALS_JSON is not valid JSON"); }+ assert(parsed !== null && typeof parsed === "object" && !Array.isArray(parsed), "Google credentials must be an object");+ return parsed as Record<string, unknown>;+}++/** Vertex Gemini VLM generator for text, image and video inline parts. */+export function createVertexGeminiGenerator(options: VertexGeminiGeneratorOptions) {+ const project = options.project.trim();+ assert(project.length > 0, "Vertex project is empty");+ const model = options.model?.trim() || "gemini-3.1-pro-preview";+ const client = new GoogleGenAI({+ vertexai: true,+ project,+ location: options.location?.trim() || "global",+ googleAuthOptions: {+ credentials: parseCredentials(options.credentials),+ scopes: ["https://www.googleapis.com/auth/cloud-platform"],+ },+ });++ return async (input: VertexGeminiGenerateInput): Promise<string> => {+ const result = await client.models.generateContent({+ model,+ contents: [{ role: "user", parts: input.parts as unknown as Content["parts"] }] as never,+ config: { systemInstruction: input.instruction, temperature: 1.0, responseMimeType: "text/plain" },+ });+ const text = result.text?.trim() ?? "";+ assert(text.length > 0, "Vertex Gemini returned an empty response");+ return text;+ };+}diff --git a/packages/provider-vertex/src/index.ts b/packages/provider-vertex/src/index.tsnew file mode 100644index 00000000..49605787--- /dev/null+++ b/packages/provider-vertex/src/index.ts@@ -0,0 +1,6 @@+export { createVertexGeminiGenerator } from "./gemini.js";+export type {+ VertexGeminiGenerateInput,+ VertexGeminiGeneratorOptions,+ VertexGeminiPart,+} from "./gemini.js";diff --git a/packages/reference-video-tools/README.md b/packages/reference-video-tools/README.mdindex aa0e2e83..a07c6993 100644--- a/packages/reference-video-tools/README.md+++ b/packages/reference-video-tools/README.md@@ -133,7 +133,8 @@ else. Results are not cached, because the rendered side changes on every iterati`unresolved` lists the keys of observations that failed. It is not a judgement about evidencequality: a complete observation that says "unclear" is still complete.-Gemini uses `HYPIT_GEMINI_PROVIDER=auto|hypihub|vertex` to select its backend. HypiHub uses+Gemini uses `HYPIT_GEMINI_PROVIDER=auto|hypihub|vertex` to select its backend. The backends are+provided by `@hypit/provider-hypihub` and `@hypit/provider-vertex`. HypiHub uses`HYPIHUB_API_KEY` and optional `HYPIHUB_BASE_URL` (default `https://hypit.ai`); Vertex uses theexisting `GOOGLE_CLOUD_PROJECT`, `GOOGLE_APPLICATION_CREDENTIALS_JSON` and optional`GOOGLE_CLOUD_LOCATION`. The model defaults to `gemini-3.1-pro-preview`. A missing or insufficientdiff --git a/packages/reference-video-tools/package.json b/packages/reference-video-tools/package.jsonindex cfec7525..76459d81 100644--- a/packages/reference-video-tools/package.json+++ b/packages/reference-video-tools/package.json@@ -12,7 +12,6 @@"./cli": "./src/cli.ts"},"dependencies": {- "@google/genai": "1.52.0","@hypit/composition": "workspace:*","@hypit/core": "workspace:*","@hypit/driver-node": "workspace:*",@@ -29,6 +28,7 @@"@hypit/provider-whisperx-local": "workspace:*","@hypit/provider-media-local": "workspace:*","@hypit/provider-hypihub": "workspace:*",+ "@hypit/provider-vertex": "workspace:*","@hypit/script": "workspace:*","@hypit/speech": "workspace:*","@hypit/speech-evidence": "workspace:*",diff --git a/packages/reference-video-tools/src/tools.ts b/packages/reference-video-tools/src/tools.tsindex 0a3bb37f..628ff16a 100644--- a/packages/reference-video-tools/src/tools.ts+++ b/packages/reference-video-tools/src/tools.ts@@ -1,4 +1,3 @@-import type { Part } from "@google/genai";import { access, appendFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";import { spawn } from "node:child_process";import { cpus } from "node:os";@@ -22,6 +21,8 @@ import { VISUAL_STYLE_ENUM_VALUES_V1, VISUAL_STYLE_NAMES_V1 } from "@hypit/visuaimport { describeSchema } from "./contract.js";import { videoCliDistribution } from "@hypit/video-cli";import { createHypiHubGeminiGenerator } from "@hypit/provider-hypihub";+import { createVertexGeminiGenerator } from "@hypit/provider-vertex";+import type { VertexGeminiPart } from "@hypit/provider-vertex";import { authorSource, invokedFrom, nearestPackageRoot, referenceRoot, referenceWords, renderElement, renderPreviews, spokenRange, standInSidecarPath, tokenWindow } from "./authoring.js";import type { RenderElementInput, RenderPreviewsInput, SpokenRange, StandInFocus, StandInSidecar } from "./authoring.js";@@ -299,7 +300,7 @@ type ToolOptions = {readonly retryDelayMs?: number;readonly generate?: GenerateText;};-export type GenerateText = (input: { readonly parts: readonly Part[]; readonly instruction: string }) => Promise<string>;+export type GenerateText = (input: { readonly parts: readonly VertexGeminiPart[]; readonly instruction: string }) => Promise<string>;type ObservationTask = { readonly key: string; readonly request: Request };function routeStateResult(state: RouteState | undefined): Record<string, unknown> {@@ -818,34 +819,13 @@ async function defaultGenerate(model: string): Promise<GenerateText> {if (!project || !credentials) {throw new Error("Gemini credentials are unavailable. Configure Vertex credentials or set HYPIHUB_API_KEY (get one at https://hypit.ai).");}- let parsed: unknown;- try { parsed = JSON.parse(credentials); } catch { throw new Error("GOOGLE_APPLICATION_CREDENTIALS_JSON is not valid JSON"); }- assert(parsed !== null && typeof parsed === "object" && !Array.isArray(parsed), "Google credentials must be an object");- let GoogleGenAI: typeof import("@google/genai")["GoogleGenAI"];- try {- ({ GoogleGenAI } = await import("@google/genai"));- } catch (error) {- throw new Error(- "Gemini observation requires @google/genai. Install it once with: hypit packages install @google/[email protected]",- { cause: error },- );- }- const client = new GoogleGenAI({- vertexai: true,+ const generate = createVertexGeminiGenerator({project,+ credentials,+ model,location: process.env.GOOGLE_CLOUD_LOCATION?.trim() || "global",- googleAuthOptions: { credentials: parsed as Record<string, unknown>, scopes: ["https://www.googleapis.com/auth/cloud-platform"] },});- return async ({ parts, instruction }) => {- const result = await client.models.generateContent({- model,- contents: [{ role: "user", parts: [...parts] }],- config: { systemInstruction: instruction, temperature: 1.0, responseMimeType: "text/plain" },- });- const text = result.text?.trim() ?? "";- assert(text.length > 0, "Gemini returned an empty response");- return text;- };+ return async ({ parts, instruction }) => generate({ parts, instruction });}const MIME_TYPES: Readonly<Record<string, string>> = {@@ -860,12 +840,12 @@ const MIME_TYPES: Readonly<Record<string, string>> = {// The Vertex path cannot upload a file and reference it later, so the bytes travel with every// request. One shot's clip is asked about by its own picture, type and sound observations and by the// boundary on each side, so reading and encoding it once per invocation is worth the memory.-function mediaParts(): (path: string) => Promise<Part> {- const encoded = new Map<string, Promise<Part>>();+function mediaParts(): (path: string) => Promise<VertexGeminiPart> {+ const encoded = new Map<string, Promise<VertexGeminiPart>>();return (path) => {const held = encoded.get(path);if (held !== undefined) return held;- const part = (async (): Promise<Part> => {+ const part = (async (): Promise<VertexGeminiPart> => {const bytes = await readBytes(path);const extension = path.toLowerCase().split(".").pop() ?? "";const mimeType = MIME_TYPES[extension];@@ -891,7 +871,7 @@ function permanent(error: unknown): boolean {return /invalid[_ ]argument|permission[_ ]denied|unauthenticated|not[_ ]found|failed[_ ]precondition/iu.test(message(error));}-async function callSafely(retryDelayMs: number, generate: GenerateText, parts: readonly Part[], instruction: string): Promise<Observation> {+async function callSafely(retryDelayMs: number, generate: GenerateText, parts: readonly VertexGeminiPart[], instruction: string): Promise<Observation> {let last: unknown;for (let attempt = 1; attempt <= 6; attempt += 1) {try { return observation("complete", await generate({ parts, instruction })); }@@ -1471,7 +1451,7 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVreturn {pending: [],ask: async (_key, { media, prompt, instruction }) => {- const parts: Part[] = [];+ const parts: VertexGeminiPart[] = [];for (const path of media) parts.push(await mediaPart(path));parts.push({ text: `${prompt}\n\n${WATERMARK_RULE}` });return await callSafely(retryDelayMs, generate, parts, instruction);diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex e6e3b0d6..c8a69ad3 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -147,6 +147,9 @@ importers:'@hypit/provider-media-local':specifier: workspace:*version: link:packages/provider-media-local+ '@hypit/provider-vertex':+ specifier: workspace:*+ version: link:packages/provider-vertex'@hypit/provider-whisperx-local':specifier: workspace:*version: link:packages/provider-whisperx-local@@ -2145,6 +2148,12 @@ importers:specifier: workspace:*version: link:../driver-node+ packages/provider-vertex:+ dependencies:+ '@google/genai':+ specifier: 1.52.0+ version: 1.52.0+packages/provider-whisperx-local:dependencies:'@hypit/endpoint-kit':@@ -2282,9 +2291,6 @@ importers:packages/reference-video-tools:dependencies:- '@google/genai':- specifier: 1.52.0- version: 1.52.0'@hypit/composition':specifier: workspace:*version: link:../composition@@ -2327,6 +2333,9 @@ importers:'@hypit/provider-media-local':specifier: workspace:*version: link:../provider-media-local+ '@hypit/provider-vertex':+ specifier: workspace:*+ version: link:../provider-vertex'@hypit/provider-whisperx-local':specifier: workspace:*version: link:../provider-whisperx-local