Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix(speech): expose only supported voice inputs

kashorin0d1b4bfd8dec
diff --git a/packages/elevenlabs-speech/README.md b/packages/elevenlabs-speech/README.md
index 09e7d918..110f05e2 100644
--- a/packages/elevenlabs-speech/README.md
+++ b/packages/elevenlabs-speech/README.md
@@ -2,9 +2,10 @@
Exact model contract and author Surface for ElevenLabs Voice Design.
-`eleven_ttv_v3` creates an ordinary audio voice reference from a natural-language description and an
-authored speech sample of 100 to 1000 characters. The service returns several candidate previews;
-the Surface exposes the primary member of the shared `GeneratedAudioSet` as `<id>.reference`.
+`eleven_ttv_v3` creates an ordinary audio voice reference from a natural-language description of
+20 to 1000 characters and an authored speech sample of 100 to 1000 characters. The service returns
+several candidate previews; the Surface exposes the primary member of the shared `GeneratedAudioSet`
+as `<id>.reference`.
It contains no API URL, credential, retry, queue or ElevenLabs wire encoding. Those belong to a
Runtime Endpoint such as `@hypit/provider-hypihub`.
diff --git a/packages/elevenlabs-speech/src/index.ts b/packages/elevenlabs-speech/src/index.ts
index 92200568..e7186d8c 100644
--- a/packages/elevenlabs-speech/src/index.ts
+++ b/packages/elevenlabs-speech/src/index.ts
@@ -12,7 +12,8 @@ export type ElevenLabsSpeechModel = typeof elevenLabsSpeechModels[number];
// ElevenLabs documents a 100 to 1000 character preview text. The port vocabulary
// carries the ceiling; the floor is stated to authors and enforced by the service.
const spokenText = { kind: "text", maxChars: 1000 } as const;
-const instruction = { kind: "text" } as const;
+// Voice descriptions have their own documented 20 to 1000 character bounds.
+const instruction = { kind: "text", maxChars: 1000 } as const;
function table(model: ElevenLabsSpeechModel): GenerationPortTable {
return sealGenerationPortTable({
@@ -95,6 +96,7 @@ export const elevenLabsSpeechMarkupSurfaces = [
notes: [
"The element accepts no child elements; only its text is read.",
"The spoken sample must be between 100 and 1000 characters.",
+ "The voice description must be between 20 and 1000 characters.",
"The result is an ordinary audio Resource that can be supplied anywhere an audio reference is accepted.",
],
},
diff --git a/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts b/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts
index 3b0d74c6..a66c8a3e 100644
--- a/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts
+++ b/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts
@@ -45,6 +45,10 @@ test("ElevenLabs Speech declares the exact voice design request shape", () => {
assert.throws(() => sealElevenLabsSpeechRequest("eleven_ttv_v3", {
text: ["Do not rewrite me."],
}), /voiceDescription is required/u);
+ assert.throws(() => sealElevenLabsSpeechRequest("eleven_ttv_v3", {
+ text: ["A voice sample with enough detail to make the test intent clear."],
+ voiceDescription: ["a".repeat(1001)],
+ }), /voiceDescription.+1000/u);
assert.equal(elevenLabsNodePackage.format, "hypit.node-package@1");
assert.deepEqual(elevenLabsSpeechMarkupSurfaces.map((surface) => surface.name), ["voiceDesign"]);
});
diff --git a/packages/fishaudio-speech/README.md b/packages/fishaudio-speech/README.md
index d15ca760..4ae888c6 100644
--- a/packages/fishaudio-speech/README.md
+++ b/packages/fishaudio-speech/README.md
@@ -19,13 +19,13 @@ Runtime Endpoint such as `@hypit/provider-hypihub`. Both models return the share
A clear young woman with a grounded, confident conversational delivery.
</fish:VoiceDesign>
-<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>
- Quietly confident, with a short pause before the final word.
-</fish:VoiceClone>
+<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}/>
```
`host.reference` is not a special identity record. It is a normal audio Resource, so the same
accepted reference can also feed an A-roll video model that accepts reference audio.
+`VoiceClone` takes spoken text and a voice reference. It has no separate natural-language delivery
+instruction input; the description in `VoiceDesign` establishes the reference voice.
Official API reference: <https://docs.fish.audio/>
diff --git a/packages/fishaudio-speech/src/index.ts b/packages/fishaudio-speech/src/index.ts
index a79f6637..ef210d25 100644
--- a/packages/fishaudio-speech/src/index.ts
+++ b/packages/fishaudio-speech/src/index.ts
@@ -15,7 +15,7 @@ export type FishAudioSpeechModel = typeof fishAudioSpeechModels[number];
// Fish Audio documents no fixed character ceiling. Service-side limits stay service-side
// instead of becoming an invented model constraint in Author Source.
const spokenText = { kind: "text" } as const;
-const instruction = { kind: "text" } as const;
+const voiceDescription = { kind: "text" } as const;
function table(model: FishAudioSpeechModel): GenerationPortTable {
if (model === "voice-design-1") {
@@ -24,7 +24,7 @@ function table(model: FishAudioSpeechModel): GenerationPortTable {
result: "audio",
ports: [
{ name: "text", value: spokenText, minItems: 1, maxItems: 1 },
- { name: "voiceDescription", value: instruction, minItems: 1, maxItems: 1 },
+ { name: "voiceDescription", value: voiceDescription, minItems: 1, maxItems: 1 },
],
requires: [],
});
@@ -34,7 +34,6 @@ function table(model: FishAudioSpeechModel): GenerationPortTable {
result: "audio",
ports: [
{ name: "text", value: spokenText, minItems: 1, maxItems: 1 },
- { name: "instruction", value: instruction, minItems: 0, maxItems: 1 },
{ name: "voiceReference", value: { kind: "media", accepts: ["audio"] }, minItems: 1, maxItems: 1 },
],
requires: [],
@@ -140,12 +139,9 @@ export const fishAudioSpeechMarkupSurfaces = [
},
],
ports: audioPort,
- text: "The element's own text is an optional delivery instruction.",
- example: `<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>
- Quietly confident, with a short pause before the final word.
-</fish:VoiceClone>`,
+ example: `<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}/>`,
notes: [
- "The element accepts no child elements; only its text is read.",
+ "The element takes its spoken words from speech and voice identity from voice; it has no body input.",
"The voice reference remains a normal audio Resource rather than a separate identity record.",
],
},
diff --git a/packages/fishaudio-speech/src/surface.ts b/packages/fishaudio-speech/src/surface.ts
index 67232642..f2496941 100644
--- a/packages/fishaudio-speech/src/surface.ts
+++ b/packages/fishaudio-speech/src/surface.ts
@@ -1,6 +1,6 @@
import { artifactTypes } from "@hypit/artifact";
import { generationPort, sealGenerationMediaBinding } from "@hypit/generation";
-import type { GenerationMediaPort, GenerationPortValue } from "@hypit/generation";
+import type { GenerationMediaPort } from "@hypit/generation";
import { exactModelMediaInputNames, exactModelTextInputName } from "@hypit/model-kit";
import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";
import type { CanonicalValue } from "@hypit/protocol";
@@ -73,7 +73,7 @@ function voiceReference(reference: SurfaceResolvedReference, subject: string): S
return reference;
}
-function body(element: StructuredElement, required: boolean): string | undefined {
+function body(element: StructuredElement): string {
if (element.children.some((child) => child.kind === "element")) {
throw new Error(`${element.name} accepts instruction text only`);
}
@@ -85,8 +85,7 @@ function body(element: StructuredElement, required: boolean): string | undefined
const indent = indents.length === 0 ? 0 : Math.min(...indents);
const value = lines.map((line) => line.slice(indent).trimEnd()).join("\n").trim();
if (value.length === 0) {
- if (required) throw new Error(`${element.name} requires a voice description in its body`);
- return undefined;
+ throw new Error(`${element.name} requires a voice description in its body`);
}
return value;
}
@@ -168,7 +167,7 @@ export const decodeFishAudioVoiceDesignSurface: StructuredSurfaceHandler = ({ el
return output(
element,
fishAudioSpeechEndpoints.voiceDesign,
- sealFishAudioSpeechRequestDraft("voice-design-1", { voiceDescription: [body(element, true)!] }),
+ sealFishAudioSpeechRequestDraft("voice-design-1", { voiceDescription: [body(element)] }),
"reference",
[speechInput(element, resolveReference)],
);
@@ -176,13 +175,13 @@ export const decodeFishAudioVoiceDesignSurface: StructuredSurfaceHandler = ({ el
export const decodeFishAudioVoiceCloneSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
attributes(element, ["id", "speech", "voice"]);
- const instruction = body(element, false);
- const ports: Record<string, readonly GenerationPortValue[]> = {};
- if (instruction !== undefined) ports.instruction = [instruction];
+ if (element.children.some((child) => child.kind === "element" || child.value.trim().length > 0)) {
+ throw new Error(`${element.name} accepts speech and voice references, with no body input`);
+ }
return output(
element,
fishAudioSpeechEndpoints.voiceClone,
- sealFishAudioSpeechRequestDraft("voice-clone", ports),
+ sealFishAudioSpeechRequestDraft("voice-clone", {}),
"audio",
[speechInput(element, resolveReference)],
[{
diff --git a/packages/fishaudio-speech/test/fishaudio-speech.test.ts b/packages/fishaudio-speech/test/fishaudio-speech.test.ts
index 08cc75df..54f46b1b 100644
--- a/packages/fishaudio-speech/test/fishaudio-speech.test.ts
+++ b/packages/fishaudio-speech/test/fishaudio-speech.test.ts
@@ -87,14 +87,19 @@ test("VoiceDesign publishes one reusable voice reference", async () => {
});
test("VoiceClone binds an accepted audio Resource and publishes independent speech", async () => {
- const result = await decodeFishAudioVoiceCloneSurface(context(`<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}>
- Calm and restrained.
- </fish:VoiceClone>`));
+ const result = await decodeFishAudioVoiceCloneSurface(context(
+ `<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}/>`));
assert.deepEqual(result.components[0]!.outputs, { audio: "narration.audio" });
assert.deepEqual(result.components[0]!.inputs["voice:artifact"], { kind: "record", id: "host.reference" });
assert.ok(result.components[0]!.inputs["voice:binding"] !== undefined);
});
+test("VoiceClone rejects a delivery body rather than silently losing it during synthesis", () => {
+ assert.throws(() => decodeFishAudioVoiceCloneSurface(context(
+ `<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}>Calm and restrained.</fish:VoiceClone>`,
+ )), /no body input/u);
+});
+
test("VoiceClone refuses a known non-audio Resource", () => {
const image: SurfaceResolvedReference = {
...voiceReference,
diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.md
index 0b7fa5dc..11396f05 100644
--- a/packages/provider-hypihub/README.md
+++ b/packages/provider-hypihub/README.md
@@ -120,7 +120,7 @@ reference to produce independent speech. All of them use `POST /v1/audio/speech`
| `@hypit/mimo-speech` | `mimo-v2.5-tts-voicedesign` | `mimo-v2.5-tts-voicedesign` | `input`, `voice_description` |
| `@hypit/mimo-speech` | `mimo-v2.5-tts-voiceclone` | `mimo-v2.5-tts-voiceclone` | `input`, `reference_audio`, optional `prompt` |
| `@hypit/fishaudio-speech` | `voice-design-1` | `fishaudio/voice-design-1` | `input`, `voice_description` |
-| `@hypit/fishaudio-speech` | `voice-clone` | `fishaudio/voice-clone` | `input`, `reference_audio`, optional `prompt`, constant `voice_description` title |
+| `@hypit/fishaudio-speech` | `voice-clone` | `fishaudio/voice-clone` | `input`, `reference_audio`, constant `voice_description` title |
| `@hypit/elevenlabs-speech` | `eleven_ttv_v3` | `eleven_ttv_v3` | `input`, `voice_description` |
Each returned preview becomes one member of the audio set. These Model packages do not expose preset
diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.ts
index f1a37f2d..40016c82 100644
--- a/packages/provider-hypihub/src/mapping.ts
+++ b/packages/provider-hypihub/src/mapping.ts
@@ -146,7 +146,6 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
constants: { voice_description: "reference" },
fields: {
text: { as: "value", field: "input" },
- instruction: { as: "value", field: "prompt" },
voiceReference: { as: "urlArray", field: "reference_audio" },
},
},
diff --git a/packages/provider-hypihub/test/provider.test.ts b/packages/provider-hypihub/test/provider.test.ts
index 521542cf..38913a57 100644
--- a/packages/provider-hypihub/test/provider.test.ts
+++ b/packages/provider-hypihub/test/provider.test.ts
@@ -7,6 +7,8 @@ import { defineEndpointPackage } from "@hypit/endpoint-kit";
import type { AsyncEndpoint } from "@hypit/endpoint-kit";
import type { CanonicalValue, Need } from "@hypit/protocol";
import { mimoSpeechEndpoints, sealMimoSpeechRequest } from "@hypit/mimo-speech";
+import { fishAudioSpeechEndpoints, sealFishAudioSpeechRequest } from "@hypit/fishaudio-speech";
+import { elevenLabsSpeechEndpoints, sealElevenLabsSpeechRequest } from "@hypit/elevenlabs-speech";
import { generationTypes } from "@hypit/generation";
import { sealSeedanceRequest, seedanceEndpoints } from "@hypit/seedance";
import { sealSpeechEvidenceAudio } from "@hypit/speech";
@@ -312,8 +314,11 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourc
const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
submitted.push(body);
const data = Buffer.from([9, 8, 7, submitted.length]).toString("base64");
- return submitted.length === 1
- ? Response.json({ object: "audio.voice_previews", previews: [{ b64_json: data, mime_type: "audio/wav" }] })
+ return cases[submitted.length - 1]!.previews
+ ? Response.json({ object: "audio.voice_previews", previews: [
+ { b64_json: data, mime_type: "audio/wav" },
+ { b64_json: Buffer.from([6, 5, 4]).toString("base64"), mime_type: "audio/wav" },
+ ] })
: Response.json({ object: "audio.speech", b64_json: data, mime_type: "audio/wav" });
},
});
@@ -325,6 +330,8 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourc
constraints: sealMimoSpeechRequest("mimo-v2.5-tts-voicedesign", {
text: ["A short voice sample."], voiceDescription: ["Warm and confident."],
}),
+ previews: true,
+ wire: { model: "mimo-v2.5-tts-voicedesign", input: "A short voice sample.", voice_description: "Warm and confident." },
},
{
endpoint: mimoSpeechEndpoints.voiceClone,
@@ -332,6 +339,37 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourc
text: ["Independent narration."], instruction: ["Quietly direct."],
voiceReference: [{ role: "audio", artifact: voiceReference }],
}),
+ previews: false,
+ wire: { model: "mimo-v2.5-tts-voiceclone", input: "Independent narration.", prompt: "Quietly direct.",
+ reference_audio: ["https://hypit.ai/assets/voice.wav"] },
+ },
+ {
+ endpoint: fishAudioSpeechEndpoints.voiceDesign,
+ constraints: sealFishAudioSpeechRequest("voice-design-1", {
+ text: ["A Fish Audio voice sample."], voiceDescription: ["A bright, playful young woman."],
+ }),
+ previews: true,
+ wire: { model: "fishaudio/voice-design-1", input: "A Fish Audio voice sample.", voice_description: "A bright, playful young woman." },
+ },
+ {
+ endpoint: fishAudioSpeechEndpoints.voiceClone,
+ constraints: sealFishAudioSpeechRequest("voice-clone", {
+ text: ["Fish Audio narration."], voiceReference: [{ role: "audio", artifact: voiceReference }],
+ }),
+ previews: false,
+ wire: { model: "fishaudio/voice-clone", input: "Fish Audio narration.", voice_description: "reference",
+ reference_audio: ["https://hypit.ai/assets/voice.wav"] },
+ },
+ {
+ endpoint: elevenLabsSpeechEndpoints.voiceDesign,
+ constraints: sealElevenLabsSpeechRequest("eleven_ttv_v3", {
+ text: ["This is a clear conversational voice sample, with enough room to hear the speaker's warmth, energy and natural rhythm."],
+ voiceDescription: ["A warm young woman with bright conversational delivery."],
+ }),
+ previews: true,
+ wire: { model: "eleven_ttv_v3",
+ input: "This is a clear conversational voice sample, with enough room to hear the speaker's warmth, energy and natural rhythm.",
+ voice_description: "A warm young woman with bright conversational delivery." },
},
];
for (const [index, item] of cases.entries()) {
@@ -354,11 +392,10 @@ test("HypiHub stores both preview JSON and ordinary speech JSON as audio Resourc
assert.equal(result.value.kind, "inline");
const set = result.value.kind === "inline" ? result.value.value as Record<string, unknown> : {};
const audios = set.audios as Array<{ resource: `res_${string}` }>;
- assert.equal(await resources.has(audios[0]!.resource), true);
+ assert.equal(audios.length, item.previews ? 2 : 1);
+ for (const audio of audios) assert.equal(await resources.has(audio.resource), true);
+ assert.deepEqual(submitted[index], { ...item.wire, output: "b64_json" });
}
- assert.equal(submitted[0]!.output, "b64_json");
- assert.deepEqual(submitted[1]!.reference_audio, ["https://hypit.ai/assets/voice.wav"]);
- assert.equal(submitted[1]!.prompt, "Quietly direct.");
});
test("HypiHub uploads one referenced Resource once and submits its HTTPS URL", async () => {