Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix(speech): expose only supported voice inputs
kashorin0d1b4bfd8dec
diff --git a/packages/elevenlabs-speech/README.md b/packages/elevenlabs-speech/README.mdindex 09e7d918..110f05e2 100644--- a/packages/elevenlabs-speech/README.md+++ b/packages/elevenlabs-speech/README.md@@ -2,9 +2,10 @@Exact model contract and author Surface for ElevenLabs Voice Design.-`eleven_ttv_v3` creates an ordinary audio voice reference from a natural-language description and an-authored speech sample of 100 to 1000 characters. The service returns several candidate previews;-the Surface exposes the primary member of the shared `GeneratedAudioSet` as `<id>.reference`.+`eleven_ttv_v3` creates an ordinary audio voice reference from a natural-language description of+20 to 1000 characters and an authored speech sample of 100 to 1000 characters. The service returns+several candidate previews; the Surface exposes the primary member of the shared `GeneratedAudioSet`+as `<id>.reference`.It contains no API URL, credential, retry, queue or ElevenLabs wire encoding. Those belong to aRuntime Endpoint such as `@hypit/provider-hypihub`.diff --git a/packages/elevenlabs-speech/src/index.ts b/packages/elevenlabs-speech/src/index.tsindex 92200568..e7186d8c 100644--- a/packages/elevenlabs-speech/src/index.ts+++ b/packages/elevenlabs-speech/src/index.ts@@ -12,7 +12,8 @@ export type ElevenLabsSpeechModel = typeof elevenLabsSpeechModels[number];// ElevenLabs documents a 100 to 1000 character preview text. The port vocabulary// carries the ceiling; the floor is stated to authors and enforced by the service.const spokenText = { kind: "text", maxChars: 1000 } as const;-const instruction = { kind: "text" } as const;+// Voice descriptions have their own documented 20 to 1000 character bounds.+const instruction = { kind: "text", maxChars: 1000 } as const;function table(model: ElevenLabsSpeechModel): GenerationPortTable {return sealGenerationPortTable({@@ -95,6 +96,7 @@ export const elevenLabsSpeechMarkupSurfaces = [notes: ["The element accepts no child elements; only its text is read.","The spoken sample must be between 100 and 1000 characters.",+ "The voice description must be between 20 and 1000 characters.","The result is an ordinary audio Resource that can be supplied anywhere an audio reference is accepted.",],},diff --git a/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts b/packages/elevenlabs-speech/test/elevenlabs-speech.test.tsindex 3b0d74c6..a66c8a3e 100644--- a/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts+++ b/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts@@ -45,6 +45,10 @@ test("ElevenLabs Speech declares the exact voice design request shape", () => {assert.throws(() => sealElevenLabsSpeechRequest("eleven_ttv_v3", {text: ["Do not rewrite me."],}), /voiceDescription is required/u);+ assert.throws(() => sealElevenLabsSpeechRequest("eleven_ttv_v3", {+ text: ["A voice sample with enough detail to make the test intent clear."],+ voiceDescription: ["a".repeat(1001)],+ }), /voiceDescription.+1000/u);assert.equal(elevenLabsNodePackage.format, "hypit.node-package@1");assert.deepEqual(elevenLabsSpeechMarkupSurfaces.map((surface) => surface.name), ["voiceDesign"]);});diff --git a/packages/fishaudio-speech/README.md b/packages/fishaudio-speech/README.mdindex d15ca760..4ae888c6 100644--- a/packages/fishaudio-speech/README.md+++ b/packages/fishaudio-speech/README.md@@ -19,13 +19,13 @@ Runtime Endpoint such as `@hypit/provider-hypihub`. Both models return the shareA clear young woman with a grounded, confident conversational delivery.</fish:VoiceDesign>-<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>- Quietly confident, with a short pause before the final word.-</fish:VoiceClone>+<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}/>````host.reference` is not a special identity record. It is a normal audio Resource, so the sameaccepted reference can also feed an A-roll video model that accepts reference audio.+`VoiceClone` takes spoken text and a voice reference. It has no separate natural-language delivery+instruction input; the description in `VoiceDesign` establishes the reference voice.Official API reference: <https://docs.fish.audio/>diff --git a/packages/fishaudio-speech/src/index.ts b/packages/fishaudio-speech/src/index.tsindex a79f6637..ef210d25 100644--- a/packages/fishaudio-speech/src/index.ts+++ b/packages/fishaudio-speech/src/index.ts@@ -15,7 +15,7 @@ export type FishAudioSpeechModel = typeof fishAudioSpeechModels[number];// Fish Audio documents no fixed character ceiling. Service-side limits stay service-side// instead of becoming an invented model constraint in Author Source.const spokenText = { kind: "text" } as const;-const instruction = { kind: "text" } as const;+const voiceDescription = { kind: "text" } as const;function table(model: FishAudioSpeechModel): GenerationPortTable {if (model === "voice-design-1") {@@ -24,7 +24,7 @@ function table(model: FishAudioSpeechModel): GenerationPortTable {result: "audio",ports: [{ name: "text", value: spokenText, minItems: 1, maxItems: 1 },- { name: "voiceDescription", value: instruction, minItems: 1, maxItems: 1 },+ { name: "voiceDescription", value: voiceDescription, minItems: 1, maxItems: 1 },],requires: [],});@@ -34,7 +34,6 @@ function table(model: FishAudioSpeechModel): GenerationPortTable {result: "audio",ports: [{ name: "text", value: spokenText, minItems: 1, maxItems: 1 },- { name: "instruction", value: instruction, minItems: 0, maxItems: 1 },{ name: "voiceReference", value: { kind: "media", accepts: ["audio"] }, minItems: 1, maxItems: 1 },],requires: [],@@ -140,12 +139,9 @@ export const fishAudioSpeechMarkupSurfaces = [},],ports: audioPort,- text: "The element's own text is an optional delivery instruction.",- example: `<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>- Quietly confident, with a short pause before the final word.-</fish:VoiceClone>`,+ example: `<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}/>`,notes: [- "The element accepts no child elements; only its text is read.",+ "The element takes its spoken words from speech and voice identity from voice; it has no body input.","The voice reference remains a normal audio Resource rather than a separate identity record.",],},diff --git a/packages/fishaudio-speech/src/surface.ts b/packages/fishaudio-speech/src/surface.tsindex 67232642..f2496941 100644--- a/packages/fishaudio-speech/src/surface.ts+++ b/packages/fishaudio-speech/src/surface.ts@@ -1,6 +1,6 @@import { artifactTypes } from "@hypit/artifact";import { generationPort, sealGenerationMediaBinding } from "@hypit/generation";-import type { GenerationMediaPort, GenerationPortValue } from "@hypit/generation";+import type { GenerationMediaPort } from "@hypit/generation";import { exactModelMediaInputNames, exactModelTextInputName } from "@hypit/model-kit";import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";import type { CanonicalValue } from "@hypit/protocol";@@ -73,7 +73,7 @@ function voiceReference(reference: SurfaceResolvedReference, subject: string): Sreturn reference;}-function body(element: StructuredElement, required: boolean): string | undefined {+function body(element: StructuredElement): string {if (element.children.some((child) => child.kind === "element")) {throw new Error(`${element.name} accepts instruction text only`);}@@ -85,8 +85,7 @@ function body(element: StructuredElement, required: boolean): string | undefinedconst indent = indents.length === 0 ? 0 : Math.min(...indents);const value = lines.map((line) => line.slice(indent).trimEnd()).join("\n").trim();if (value.length === 0) {- if (required) throw new Error(`${element.name} requires a voice description in its body`);- return undefined;+ throw new Error(`${element.name} requires a voice description in its body`);}return value;}@@ -168,7 +167,7 @@ export const decodeFishAudioVoiceDesignSurface: StructuredSurfaceHandler = ({ elreturn output(element,fishAudioSpeechEndpoints.voiceDesign,- sealFishAudioSpeechRequestDraft("voice-design-1", { voiceDescription: [body(element, true)!] }),+ sealFishAudioSpeechRequestDraft("voice-design-1", { voiceDescription: [body(element)] }),"reference",[speechInput(element, resolveReference)],);@@ -176,13 +175,13 @@ export const decodeFishAudioVoiceDesignSurface: StructuredSurfaceHandler = ({ elexport const decodeFishAudioVoiceCloneSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {attributes(element, ["id", "speech", "voice"]);- const instruction = body(element, false);- const ports: Record<string, readonly GenerationPortValue[]> = {};- if (instruction !== undefined) ports.instruction = [instruction];+ if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {+ throw new Error(`${element.name} accepts speech and voice references, with no body input`);+ }return output(element,fishAudioSpeechEndpoints.voiceClone,- sealFishAudioSpeechRequestDraft("voice-clone", ports),+ sealFishAudioSpeechRequestDraft("voice-clone", {}),"audio",[speechInput(element, resolveReference)],[{diff --git a/packages/fishaudio-speech/test/fishaudio-speech.test.ts b/packages/fishaudio-speech/test/fishaudio-speech.test.tsindex 08cc75df..54f46b1b 100644--- a/packages/fishaudio-speech/test/fishaudio-speech.test.ts+++ b/packages/fishaudio-speech/test/fishaudio-speech.test.ts@@ -87,14 +87,19 @@ test("VoiceDesign publishes one reusable voice reference", async () => {});test("VoiceClone binds an accepted audio Resource and publishes independent speech", async () => {- const result = await decodeFishAudioVoiceCloneSurface(context(`<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}>- Calm and restrained.- </fish:VoiceClone>`));+ const result = await decodeFishAudioVoiceCloneSurface(context(+ `<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}/>`));assert.deepEqual(result.components[0]!.outputs, { audio: "narration.audio" });assert.deepEqual(result.components[0]!.inputs["voice:artifact"], { kind: "record", id: "host.reference" });assert.ok(result.components[0]!.inputs["voice:binding"] !== undefined);});+test("VoiceClone rejects a delivery body rather than silently losing it during synthesis", () => {+ assert.throws(() => decodeFishAudioVoiceCloneSurface(context(+ `<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}>Calm and restrained.</fish:VoiceClone>`,+ )), /no body input/u);+});+test("VoiceClone refuses a known non-audio Resource", () => {const image: SurfaceResolvedReference = {...voiceReference,diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.mdindex 0b7fa5dc..11396f05 100644--- a/packages/provider-hypihub/README.md+++ b/packages/provider-hypihub/README.md@@ -120,7 +120,7 @@ reference to produce independent speech. All of them use `POST /v1/audio/speech`| `@hypit/mimo-speech` | `mimo-v2.5-tts-voicedesign` | `mimo-v2.5-tts-voicedesign` | `input`, `voice_description` || `@hypit/mimo-speech` | `mimo-v2.5-tts-voiceclone` | `mimo-v2.5-tts-voiceclone` | `input`, `reference_audio`, optional `prompt` || `@hypit/fishaudio-speech` | `voice-design-1` | `fishaudio/voice-design-1` | `input`, `voice_description` |-| `@hypit/fishaudio-speech` | `voice-clone` | `fishaudio/voice-clone` | `input`, `reference_audio`, optional `prompt`, constant `voice_description` title |+| `@hypit/fishaudio-speech` | `voice-clone` | `fishaudio/voice-clone` | `input`, `reference_audio`, constant `voice_description` title || `@hypit/elevenlabs-speech` | `eleven_ttv_v3` | `eleven_ttv_v3` | `input`, `voice_description` |Each returned preview becomes one member of the audio set. These Model packages do not expose presetdiff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.tsindex f1a37f2d..40016c82 100644--- a/packages/provider-hypihub/src/mapping.ts+++ b/packages/provider-hypihub/src/mapping.ts@@ -146,7 +146,6 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [constants: { voice_description: "reference" },fields: {text: { as: "value", field: "input" },- instruction: { as: "value", field: "prompt" },voiceReference: { as: "urlArray", field: "reference_audio" },},},diff --git a/packages/provider-hypihub/test/provider.test.ts b/packages/provider-hypihub/test/provider.test.tsindex 521542cf..38913a57 100644--- a/packages/provider-hypihub/test/provider.test.ts+++ b/packages/provider-hypihub/test/provider.test.ts@@ -7,6 +7,8 @@ import { defineEndpointPackage } from "@hypit/endpoint-kit";import type { AsyncEndpoint } from "@hypit/endpoint-kit";import type { CanonicalValue, Need } from "@hypit/protocol";import { mimoSpeechEndpoints, sealMimoSpeechRequest } from "@hypit/mimo-speech";+import { fishAudioSpeechEndpoints, sealFishAudioSpeechRequest } from "@hypit/fishaudio-speech";+import { elevenLabsSpeechEndpoints, sealElevenLabsSpeechRequest } from "@hypit/elevenlabs-speech";import { generationTypes } from "@hypit/generation";import { sealSeedanceRequest, seedanceEndpoints } from "@hypit/seedance";import { sealSpeechEvidenceAudio } from "@hypit/speech";@@ -312,8 +314,11 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourcconst body = JSON.parse(String(init?.body)) as Record<string, unknown>;submitted.push(body);const data = Buffer.from([9, 8, 7, submitted.length]).toString("base64");- return submitted.length === 1- ? Response.json({ object: "audio.voice_previews", previews: [{ b64_json: data, mime_type: "audio/wav" }] })+ return cases[submitted.length - 1]!.previews+ ? Response.json({ object: "audio.voice_previews", previews: [+ { b64_json: data, mime_type: "audio/wav" },+ { b64_json: Buffer.from([6, 5, 4]).toString("base64"), mime_type: "audio/wav" },+ ] }): Response.json({ object: "audio.speech", b64_json: data, mime_type: "audio/wav" });},});@@ -325,6 +330,8 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourcconstraints: sealMimoSpeechRequest("mimo-v2.5-tts-voicedesign", {text: ["A short voice sample."], voiceDescription: ["Warm and confident."],}),+ previews: true,+ wire: { model: "mimo-v2.5-tts-voicedesign", input: "A short voice sample.", voice_description: "Warm and confident." },},{endpoint: mimoSpeechEndpoints.voiceClone,@@ -332,6 +339,37 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourctext: ["Independent narration."], instruction: ["Quietly direct."],voiceReference: [{ role: "audio", artifact: voiceReference }],}),+ previews: false,+ wire: { model: "mimo-v2.5-tts-voiceclone", input: "Independent narration.", prompt: "Quietly direct.",+ reference_audio: ["https://hypit.ai/assets/voice.wav"] },+ },+ {+ endpoint: fishAudioSpeechEndpoints.voiceDesign,+ constraints: sealFishAudioSpeechRequest("voice-design-1", {+ text: ["A Fish Audio voice sample."], voiceDescription: ["A bright, playful young woman."],+ }),+ previews: true,+ wire: { model: "fishaudio/voice-design-1", input: "A Fish Audio voice sample.", voice_description: "A bright, playful young woman." },+ },+ {+ endpoint: fishAudioSpeechEndpoints.voiceClone,+ constraints: sealFishAudioSpeechRequest("voice-clone", {+ text: ["Fish Audio narration."], voiceReference: [{ role: "audio", artifact: voiceReference }],+ }),+ previews: false,+ wire: { model: "fishaudio/voice-clone", input: "Fish Audio narration.", voice_description: "reference",+ reference_audio: ["https://hypit.ai/assets/voice.wav"] },+ },+ {+ endpoint: elevenLabsSpeechEndpoints.voiceDesign,+ constraints: sealElevenLabsSpeechRequest("eleven_ttv_v3", {+ text: ["This is a clear conversational voice sample, with enough room to hear the speaker's warmth, energy and natural rhythm."],+ voiceDescription: ["A warm young woman with bright conversational delivery."],+ }),+ previews: true,+ wire: { model: "eleven_ttv_v3",+ input: "This is a clear conversational voice sample, with enough room to hear the speaker's warmth, energy and natural rhythm.",+ voice_description: "A warm young woman with bright conversational delivery." },},];for (const [index, item] of cases.entries()) {@@ -354,11 +392,10 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourcassert.equal(result.value.kind, "inline");const set = result.value.kind === "inline" ? result.value.value as Record<string, unknown> : {};const audios = set.audios as Array<{ resource: `res_${string}` }>;- assert.equal(await resources.has(audios[0]!.resource), true);+ assert.equal(audios.length, item.previews ? 2 : 1);+ for (const audio of audios) assert.equal(await resources.has(audio.resource), true);+ assert.deepEqual(submitted[index], { ...item.wire, output: "b64_json" });}- assert.equal(submitted[0]!.output, "b64_json");- assert.deepEqual(submitted[1]!.reference_audio, ["https://hypit.ai/assets/voice.wav"]);- assert.equal(submitted[1]!.prompt, "Quietly direct.");});test("HypiHub uploads one referenced Resource once and submits its HTTPS URL", async () => {