Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat(speech): add Fish Audio and ElevenLabs speech packages

rpone594bd89092c0
diff --git a/package.json b/package.json
index c2c9f475..1490a85c 100644
--- a/package.json
+++ b/package.json
@@ -189,9 +189,11 @@
"@hypit/credential-store-os": "workspace:*",
"@hypit/deck-track": "workspace:*",
"@hypit/deck-track-studio": "workspace:*",
+ "@hypit/elevenlabs-speech": "workspace:*",
"@hypit/estimate": "workspace:*",
"@hypit/film": "workspace:*",
"@hypit/film-studio": "workspace:*",
+ "@hypit/fishaudio-speech": "workspace:*",
"@hypit/fonts-open": "workspace:*",
"@hypit/generation": "workspace:*",
"@hypit/gpt-image": "workspace:*",
diff --git a/packages/elevenlabs-speech/README.md b/packages/elevenlabs-speech/README.md
new file mode 100644
index 00000000..e76f47f5
--- /dev/null
+++ b/packages/elevenlabs-speech/README.md
@@ -0,0 +1,31 @@
+# `@hypit/elevenlabs-speech`
+
+Exact model contract and author Surface for ElevenLabs Voice Design.
+
+`eleven_ttv_v3` creates an ordinary audio voice reference from a natural-language description and an
+authored speech sample of 100 to 1000 characters. The service returns several candidate previews;
+the Surface exposes the primary member of the shared `GeneratedAudioSet` as `<id>.reference`.
+
+It contains no API URL, credential, retry, queue or ElevenLabs wire encoding. Those belong to a
+Runtime Endpoint such as `@hypit/provider-hypihub`.
+
+```xml
+<import as="eleven" from="@hypit/elevenlabs-speech@1"/>
+<import as="mimo" from="@hypit/mimo-speech@1"/>
+
+<eleven:VoiceDesign id="host" speech={story.segment.voiceSample.speech}>
+ A clear young woman with a grounded, confident conversational delivery.
+</eleven:VoiceDesign>
+
+<mimo:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>
+ Quietly confident, with a short pause before the final word.
+</mimo:VoiceClone>
+```
+
+`host.reference` is a normal audio Resource, so it can feed a Voice Clone model from another
+package or an A-roll video model that accepts reference audio.
+
+Official API reference: <https://elevenlabs.io/docs/api-reference/text-to-voice/design>
+
+The Source import selects author syntax and exact model semantics only. The Runtime Profile
+independently selects the Provider that fulfils the capability.
diff --git a/packages/elevenlabs-speech/package.json b/packages/elevenlabs-speech/package.json
new file mode 100644
index 00000000..9ff9cc73
--- /dev/null
+++ b/packages/elevenlabs-speech/package.json
@@ -0,0 +1,21 @@
+{
+ "name": "@hypit/elevenlabs-speech",
+ "version": "0.0.0-dev",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@hypit/artifact": "workspace:*",
+ "@hypit/generation": "workspace:*",
+ "@hypit/model-kit": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/text": "workspace:*",
+ "@hypit/markup": "workspace:*"
+ }
+}
diff --git a/packages/elevenlabs-speech/src/activation.ts b/packages/elevenlabs-speech/src/activation.ts
new file mode 100644
index 00000000..363b63d5
--- /dev/null
+++ b/packages/elevenlabs-speech/src/activation.ts
@@ -0,0 +1,26 @@
+import { createMarkupSurfaceHostFacet } from "@hypit/markup";
+
+import {
+ decodeElevenLabsVoiceDesignSurface,
+ elevenLabsSpeechComponent,
+ elevenLabsSpeechDefinition,
+ elevenLabsSpeechManifest,
+ elevenLabsSpeechMarkupSurfaces,
+ elevenLabsSpeechModuleRef,
+} from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.node-package@1" as const,
+ modules: [{ manifest: elevenLabsSpeechManifest }],
+ components: [elevenLabsSpeechComponent],
+ hostFacets: [
+ elevenLabsSpeechDefinition.hostFacet,
+ createMarkupSurfaceHostFacet({
+ module: elevenLabsSpeechModuleRef,
+ declaration: elevenLabsSpeechMarkupSurfaces.find((item) => item.name === "voiceDesign")!,
+ handler: decodeElevenLabsVoiceDesignSurface,
+ }),
+ ],
+};
+
+export default hypitPackage;
diff --git a/packages/elevenlabs-speech/src/fragment.ts b/packages/elevenlabs-speech/src/fragment.ts
new file mode 100644
index 00000000..7204e895
--- /dev/null
+++ b/packages/elevenlabs-speech/src/fragment.ts
@@ -0,0 +1,11 @@
+import { createExactModelPrimaryGenerationFragment } from "@hypit/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";
+
+/** Model result projection over the same graph-native request assembly as every exact model. */
+export function createElevenLabsSpeechAudioFragment(
+ endpoint: ExactModelEndpoint,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+ textInputs: readonly ExactModelTextInput[] = [],
+) {
+ return createExactModelPrimaryGenerationFragment(endpoint, mediaInputs, textInputs);
+}
diff --git a/packages/elevenlabs-speech/src/index.ts b/packages/elevenlabs-speech/src/index.ts
new file mode 100644
index 00000000..bcc5f6d0
--- /dev/null
+++ b/packages/elevenlabs-speech/src/index.ts
@@ -0,0 +1,109 @@
+import { artifactTypes } from "@hypit/artifact";
+import { sealGenerationPortRequest, sealGenerationRequestDraft, sealGenerationPortTable } from "@hypit/generation";
+import type { GenerationPortTable, GenerationPortValue, GenerationRequest } from "@hypit/generation";
+import type { SurfaceAttributeVocabulary, SurfacePortVocabulary } from "@hypit/markup";
+import { defineExactModelModule } from "@hypit/model-kit";
+import { textTypes } from "@hypit/text";
+
+export const elevenLabsSpeechModuleRef = { name: "@hypit/elevenlabs-speech", version: "1" } as const;
+export const elevenLabsSpeechModels = ["eleven_ttv_v3"] as const;
+export type ElevenLabsSpeechModel = typeof elevenLabsSpeechModels[number];
+
+// ElevenLabs documents a 100 to 1000 character preview text. The port vocabulary
+// carries the ceiling; the floor is stated to authors and enforced by the service.
+const spokenText = { kind: "text", maxChars: 1000 } as const;
+const instruction = { kind: "text" } as const;
+
+function table(model: ElevenLabsSpeechModel): GenerationPortTable {
+ return sealGenerationPortTable({
+ model,
+ result: "audio",
+ ports: [
+ { name: "text", value: spokenText, minItems: 1, maxItems: 1 },
+ { name: "voiceDescription", value: instruction, minItems: 1, maxItems: 1 },
+ ],
+ requires: [],
+ });
+}
+
+export const elevenLabsSpeechPorts: Readonly<Record<ElevenLabsSpeechModel, GenerationPortTable>> = {
+ "eleven_ttv_v3": table("eleven_ttv_v3"),
+};
+
+export function sealElevenLabsSpeechRequest(
+ model: ElevenLabsSpeechModel,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+): GenerationRequest {
+ return sealGenerationPortRequest(elevenLabsSpeechPorts[model], ports);
+}
+
+export function sealElevenLabsSpeechRequestDraft(
+ model: ElevenLabsSpeechModel,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+) {
+ return sealGenerationRequestDraft(elevenLabsSpeechPorts[model], ports);
+}
+
+const base = defineExactModelModule({
+ module: elevenLabsSpeechModuleRef,
+ endpoints: ([
+ ["voiceDesign", "eleven_ttv_v3", "ElevenLabsVoiceDesignRequest"],
+ ] as const).map(([key, model, requestTypeName]) => ({
+ key,
+ requestTypeName,
+ producerName: `request-${model}`,
+ ports: elevenLabsSpeechPorts[model],
+ })),
+});
+
+export const elevenLabsSpeechEndpoints = base.endpoints;
+
+const spokenAttributes: readonly SurfaceAttributeVocabulary[] = [
+ {
+ name: "id",
+ kind: "identifier",
+ required: true,
+ summary: "Names this speech request and prefixes the binding it publishes.",
+ },
+ {
+ name: "speech",
+ kind: "reference",
+ required: true,
+ summary: "The Text edge whose exact words are spoken.",
+ accepts: [textTypes.text],
+ },
+];
+
+const referencePort: readonly SurfacePortVocabulary[] = [{
+ name: "reference",
+ type: artifactTypes.blob,
+ summary: "The designed voice reference, addressed as `<id>.reference`.",
+}];
+
+export const elevenLabsSpeechMarkupSurfaces = [
+ {
+ name: "voiceDesign", tag: "VoiceDesign", mode: "structured",
+ outputs: [elevenLabsSpeechEndpoints.voiceDesign.draftType],
+ vocabulary: {
+ summary: "Creates a reusable voice reference from a natural-language voice description.",
+ attributes: spokenAttributes,
+ ports: referencePort,
+ text: "The element's own text is the voice description and is required.",
+ example: `<eleven:VoiceDesign id="host" speech={story.segment.voiceSample.speech}>
+ A clear young woman with a grounded, confident conversational delivery.
+</eleven:VoiceDesign>`,
+ notes: [
+ "The element accepts no child elements; only its text is read.",
+ "The spoken sample must be between 100 and 1000 characters.",
+ "The result is an ordinary audio Resource that can be supplied anywhere an audio reference is accepted, including a Voice Clone model from another package.",
+ ],
+ },
+ },
+] as const;
+
+export const elevenLabsSpeechManifest = { ...base.manifest } as const;
+export const elevenLabsSpeechComponent = base.component;
+export const elevenLabsSpeechDefinition = { ...base, manifest: elevenLabsSpeechManifest };
+
+export { createElevenLabsSpeechAudioFragment } from "./fragment.js";
+export { decodeElevenLabsVoiceDesignSurface } from "./surface.js";
diff --git a/packages/elevenlabs-speech/src/surface.ts b/packages/elevenlabs-speech/src/surface.ts
new file mode 100644
index 00000000..3581ce30
--- /dev/null
+++ b/packages/elevenlabs-speech/src/surface.ts
@@ -0,0 +1,168 @@
+import { artifactTypes } from "@hypit/artifact";
+import { generationPort, sealGenerationMediaBinding } from "@hypit/generation";
+import type { GenerationMediaPort } from "@hypit/generation";
+import { exactModelMediaInputNames, exactModelTextInputName } from "@hypit/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";
+import type { CanonicalValue } from "@hypit/protocol";
+import { textTypes, verifyText } from "@hypit/text";
+import type {
+ MarkupAttributeValue,
+ StructuredElement,
+ StructuredSurfaceHandler,
+ SurfaceResolvedReference,
+} from "@hypit/markup";
+
+import { createElevenLabsSpeechAudioFragment } from "./fragment.js";
+import { elevenLabsSpeechEndpoints, sealElevenLabsSpeechRequestDraft } from "./index.js";
+
+function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
+ return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
+}
+
+function attributes(element: StructuredElement, required: readonly string[]): void {
+ const allowed = new Set(required);
+ const unknown = Object.keys(element.attributes).filter((name) => !allowed.has(name));
+ if (unknown.length > 0 || required.some((name) => element.attributes[name] === undefined)) {
+ throw new Error(`${element.name} requires ${required.join(", ")}`);
+ }
+}
+
+function stringAttribute(element: StructuredElement, name: string): string {
+ const value = element.attributes[name];
+ if (typeof value !== "string" || value.trim().length === 0) {
+ throw new Error(`${element.name}.${name} must be a non-empty string`);
+ }
+ return value.trim();
+}
+
+function referencePath(element: StructuredElement, name: string): string {
+ const value: MarkupAttributeValue | undefined = element.attributes[name];
+ if (typeof value !== "object" || value.kind !== "reference" || value.path.length === 0) {
+ throw new Error(`${element.name}.${name} must be a whole-value reference`);
+ }
+ return value.path;
+}
+
+function resolved(
+ element: StructuredElement,
+ name: string,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ const path = referencePath(element, name);
+ const value = resolveReference(path);
+ if (value === undefined) throw new Error(`${element.name}.${name} cannot resolve ${path}`);
+ return value;
+}
+
+function speechText(reference: SurfaceResolvedReference, subject: string): SurfaceResolvedReference {
+ if (!sameType(reference.type, textTypes.text)) throw new Error(`${subject} must reference Text`);
+ const value = reference.record?.value;
+ if (value !== undefined) {
+ if (value.kind !== "inline") throw new Error(`${subject} has an invalid authored Text value`);
+ verifyText(value.value);
+ }
+ return reference;
+}
+
+
+function body(element: StructuredElement, required: boolean): string | undefined {
+ if (element.children.some((child) => child.kind === "element")) {
+ throw new Error(`${element.name} accepts instruction text only`);
+ }
+ const lines = element.children.map((child) => child.kind === "text" ? child.value : "").join("")
+ .replaceAll("\r\n", "\n").split("\n");
+ while (lines[0]?.trim() === "") lines.shift();
+ while (lines.at(-1)?.trim() === "") lines.pop();
+ const indents = lines.filter((line) => line.trim()).map((line) => /^\s*/u.exec(line)?.[0].length ?? 0);
+ const indent = indents.length === 0 ? 0 : Math.min(...indents);
+ const value = lines.map((line) => line.slice(indent).trimEnd()).join("\n").trim();
+ if (value.length === 0) {
+ if (required) throw new Error(`${element.name} requires a voice description in its body`);
+ return undefined;
+ }
+ return value;
+}
+
+function output(
+ element: StructuredElement,
+ endpoint: ExactModelEndpoint,
+ draft: ReturnType<typeof sealElevenLabsSpeechRequestDraft>,
+ outputName: "reference" | "audio",
+ text: readonly { readonly input: ExactModelTextInput; readonly source: SurfaceResolvedReference }[],
+ media: readonly { readonly input: ExactModelMediaInput; readonly source: SurfaceResolvedReference }[] = [],
+) {
+ const id = stringAttribute(element, "id");
+ const draftId = `${id}.draft`;
+ const records: Array<{
+ readonly id: string;
+ readonly type: SurfaceResolvedReference["type"];
+ readonly value: { readonly kind: "inline"; readonly value: CanonicalValue };
+ readonly range: StructuredElement["range"];
+ }> = [{
+ id: draftId,
+ type: endpoint.draftType,
+ value: { kind: "inline", value: draft as unknown as CanonicalValue },
+ range: element.range,
+ }];
+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }> = {
+ draft: { kind: "record", id: draftId },
+ };
+ for (const item of text) inputs[exactModelTextInputName(item.input.name)] = item.source.ref;
+ for (const item of media) {
+ const binding = endpoint.mediaBindings[item.input.port];
+ if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${item.input.port}`);
+ const port = generationPort(endpoint.ports, item.input.port);
+ if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${item.input.port} is not media`);
+ const names = exactModelMediaInputNames(item.input.name);
+ const bindingId = `${id}.${item.input.name}.binding`;
+ records.push({
+ id: bindingId,
+ type: binding.type,
+ value: {
+ kind: "inline",
+ value: sealGenerationMediaBinding(port as GenerationMediaPort, { role: "audio" }) as unknown as CanonicalValue,
+ },
+ range: element.range,
+ });
+ inputs[names.binding] = { kind: "record", id: bindingId };
+ inputs[names.artifact] = item.source.ref;
+ }
+ const fragment = createElevenLabsSpeechAudioFragment(
+ endpoint,
+ media.map((item) => item.input),
+ text.map((item) => item.input),
+ );
+ return {
+ records,
+ components: [{
+ id,
+ fragment: fragment.id,
+ inputs,
+ outputs: { audio: `${id}.${outputName}` },
+ range: element.range,
+ }],
+ fragments: [fragment],
+ };
+}
+
+function speechInput(
+ element: StructuredElement,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+) {
+ return {
+ input: { name: "speech", port: "text" },
+ source: speechText(resolved(element, "speech", resolveReference), `${element.name}.speech`),
+ } as const;
+}
+
+export const decodeElevenLabsVoiceDesignSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ attributes(element, ["id", "speech"]);
+ return output(
+ element,
+ elevenLabsSpeechEndpoints.voiceDesign,
+ sealElevenLabsSpeechRequestDraft("eleven_ttv_v3", { voiceDescription: [body(element, true)!] }),
+ "reference",
+ [speechInput(element, resolveReference)],
+ );
+};
+
diff --git a/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts b/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts
new file mode 100644
index 00000000..3b0d74c6
--- /dev/null
+++ b/packages/elevenlabs-speech/test/elevenlabs-speech.test.ts
@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { artifactTypes } from "@hypit/artifact";
+import { generationTypes } from "@hypit/generation";
+import { parseStructuredElement } from "@hypit/markup";
+import type { SurfaceResolvedReference } from "@hypit/markup";
+import { sealText, textTypes } from "@hypit/text";
+
+import elevenLabsNodePackage from "../src/activation.js";
+import {
+ decodeElevenLabsVoiceDesignSurface,
+ elevenLabsSpeechEndpoints,
+ elevenLabsSpeechMarkupSurfaces,
+ elevenLabsSpeechModels,
+ sealElevenLabsSpeechRequest,
+} from "../src/index.js";
+
+function parsed(source: string) {
+ return parseStructuredElement({ name: "eleven.svml", text: source }, 0).element;
+}
+
+const speech = sealText("Exact authored words stay exact.");
+const speechReference: SurfaceResolvedReference = {
+ path: "story.segment.opening.speech",
+ ref: { kind: "record", id: "story.segment.opening.speech" },
+ type: textTypes.text,
+ record: {
+ id: "story.segment.opening.speech",
+ type: textTypes.text,
+ value: { kind: "inline", value: speech },
+ },
+};
+
+const context = (source: string) => ({
+ sourceName: "eleven.svml",
+ element: parsed(source),
+ resolveReference: (path: string) => path === speechReference.path ? speechReference : undefined,
+ resolveAsset: () => { throw new Error("no asset"); },
+});
+
+test("ElevenLabs Speech declares the exact voice design request shape", () => {
+ assert.deepEqual(elevenLabsSpeechModels, ["eleven_ttv_v3"]);
+ assert.equal(elevenLabsSpeechEndpoints.voiceDesign.returns.name, generationTypes.audioSet.name);
+ assert.throws(() => sealElevenLabsSpeechRequest("eleven_ttv_v3", {
+ text: ["Do not rewrite me."],
+ }), /voiceDescription is required/u);
+ assert.equal(elevenLabsNodePackage.format, "hypit.node-package@1");
+ assert.deepEqual(elevenLabsSpeechMarkupSurfaces.map((surface) => surface.name), ["voiceDesign"]);
+});
+
+test("VoiceDesign publishes one reusable voice reference", async () => {
+ const result = await decodeElevenLabsVoiceDesignSurface(context(`<eleven:VoiceDesign id="host" speech={story.segment.opening.speech}>
+ A clear, confident young woman with a grounded conversational tone.
+ </eleven:VoiceDesign>`));
+ assert.deepEqual(result.components[0]!.outputs, { audio: "host.reference" });
+ const exported = result.fragments[0]!.exports.find((item) => item.name === "audio");
+ assert.deepEqual(exported?.type, artifactTypes.blob);
+ assert.deepEqual(result.components[0]!.inputs["speech:text"], {
+ kind: "record",
+ id: "story.segment.opening.speech",
+ });
+});
diff --git a/packages/fishaudio-speech/README.md b/packages/fishaudio-speech/README.md
new file mode 100644
index 00000000..d15ca760
--- /dev/null
+++ b/packages/fishaudio-speech/README.md
@@ -0,0 +1,33 @@
+# `@hypit/fishaudio-speech`
+
+Exact model contracts and author Surfaces for Fish Audio Voice Design and Voice Clone.
+
+The package owns two distinct speech operations:
+
+- `voice-design-1` creates an ordinary audio voice reference from a natural-language description
+ and a short authored speech sample;
+- `voice-clone` uses one accepted audio voice reference to create independent speech.
+
+It contains no API URL, credential, retry, queue or Fish Audio wire encoding. Those belong to a
+Runtime Endpoint such as `@hypit/provider-hypihub`. Both models return the shared
+`GeneratedAudioSet`; the author Surfaces expose its primary member as an ordinary audio Resource.
+
+```xml
+<import as="fish" from="@hypit/fishaudio-speech@1"/>
+
+<fish:VoiceDesign id="host" speech={story.segment.voiceSample.speech}>
+ A clear young woman with a grounded, confident conversational delivery.
+</fish:VoiceDesign>
+
+<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>
+ Quietly confident, with a short pause before the final word.
+</fish:VoiceClone>
+```
+
+`host.reference` is not a special identity record. It is a normal audio Resource, so the same
+accepted reference can also feed an A-roll video model that accepts reference audio.
+
+Official API reference: <https://docs.fish.audio/>
+
+The Source import selects author syntax and exact model semantics only. The Runtime Profile
+independently selects the Provider that fulfils either capability.
diff --git a/packages/fishaudio-speech/package.json b/packages/fishaudio-speech/package.json
new file mode 100644
index 00000000..871cb8c8
--- /dev/null
+++ b/packages/fishaudio-speech/package.json
@@ -0,0 +1,21 @@
+{
+ "name": "@hypit/fishaudio-speech",
+ "version": "0.0.0-dev",
+ "license": "SEE LICENSE IN LICENSE",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "hypit": {
+ "activation": "./src/activation.ts"
+ },
+ "dependencies": {
+ "@hypit/artifact": "workspace:*",
+ "@hypit/generation": "workspace:*",
+ "@hypit/model-kit": "workspace:*",
+ "@hypit/protocol": "workspace:*",
+ "@hypit/text": "workspace:*",
+ "@hypit/markup": "workspace:*"
+ }
+}
diff --git a/packages/fishaudio-speech/src/activation.ts b/packages/fishaudio-speech/src/activation.ts
new file mode 100644
index 00000000..383328e9
--- /dev/null
+++ b/packages/fishaudio-speech/src/activation.ts
@@ -0,0 +1,32 @@
+import { createMarkupSurfaceHostFacet } from "@hypit/markup";
+
+import {
+ decodeFishAudioVoiceCloneSurface,
+ decodeFishAudioVoiceDesignSurface,
+ fishAudioSpeechComponent,
+ fishAudioSpeechDefinition,
+ fishAudioSpeechManifest,
+ fishAudioSpeechMarkupSurfaces,
+ fishAudioSpeechModuleRef,
+} from "./index.js";
+
+export const hypitPackage = {
+ format: "hypit.node-package@1" as const,
+ modules: [{ manifest: fishAudioSpeechManifest }],
+ components: [fishAudioSpeechComponent],
+ hostFacets: [
+ fishAudioSpeechDefinition.hostFacet,
+ createMarkupSurfaceHostFacet({
+ module: fishAudioSpeechModuleRef,
+ declaration: fishAudioSpeechMarkupSurfaces.find((item) => item.name === "voiceDesign")!,
+ handler: decodeFishAudioVoiceDesignSurface,
+ }),
+ createMarkupSurfaceHostFacet({
+ module: fishAudioSpeechModuleRef,
+ declaration: fishAudioSpeechMarkupSurfaces.find((item) => item.name === "voiceClone")!,
+ handler: decodeFishAudioVoiceCloneSurface,
+ }),
+ ],
+};
+
+export default hypitPackage;
diff --git a/packages/fishaudio-speech/src/fragment.ts b/packages/fishaudio-speech/src/fragment.ts
new file mode 100644
index 00000000..068b7cc7
--- /dev/null
+++ b/packages/fishaudio-speech/src/fragment.ts
@@ -0,0 +1,11 @@
+import { createExactModelPrimaryGenerationFragment } from "@hypit/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";
+
+/** Model result projection over the same graph-native request assembly as every exact model. */
+export function createFishAudioSpeechAudioFragment(
+ endpoint: ExactModelEndpoint,
+ mediaInputs: readonly ExactModelMediaInput[] = [],
+ textInputs: readonly ExactModelTextInput[] = [],
+) {
+ return createExactModelPrimaryGenerationFragment(endpoint, mediaInputs, textInputs);
+}
diff --git a/packages/fishaudio-speech/src/index.ts b/packages/fishaudio-speech/src/index.ts
new file mode 100644
index 00000000..a79f6637
--- /dev/null
+++ b/packages/fishaudio-speech/src/index.ts
@@ -0,0 +1,160 @@
+import { artifactTypes } from "@hypit/artifact";
+import { sealGenerationPortRequest, sealGenerationRequestDraft, sealGenerationPortTable } from "@hypit/generation";
+import type { GenerationPortTable, GenerationPortValue, GenerationRequest } from "@hypit/generation";
+import type { SurfaceAttributeVocabulary, SurfacePortVocabulary } from "@hypit/markup";
+import { defineExactModelModule } from "@hypit/model-kit";
+import { textTypes } from "@hypit/text";
+
+export const fishAudioSpeechModuleRef = { name: "@hypit/fishaudio-speech", version: "1" } as const;
+export const fishAudioSpeechModels = [
+ "voice-design-1",
+ "voice-clone",
+] as const;
+export type FishAudioSpeechModel = typeof fishAudioSpeechModels[number];
+
+// Fish Audio documents no fixed character ceiling. Service-side limits stay service-side
+// instead of becoming an invented model constraint in Author Source.
+const spokenText = { kind: "text" } as const;
+const instruction = { kind: "text" } as const;
+
+function table(model: FishAudioSpeechModel): GenerationPortTable {
+ if (model === "voice-design-1") {
+ return sealGenerationPortTable({
+ model,
+ result: "audio",
+ ports: [
+ { name: "text", value: spokenText, minItems: 1, maxItems: 1 },
+ { name: "voiceDescription", value: instruction, minItems: 1, maxItems: 1 },
+ ],
+ requires: [],
+ });
+ }
+ return sealGenerationPortTable({
+ model,
+ result: "audio",
+ ports: [
+ { name: "text", value: spokenText, minItems: 1, maxItems: 1 },
+ { name: "instruction", value: instruction, minItems: 0, maxItems: 1 },
+ { name: "voiceReference", value: { kind: "media", accepts: ["audio"] }, minItems: 1, maxItems: 1 },
+ ],
+ requires: [],
+ });
+}
+
+export const fishAudioSpeechPorts: Readonly<Record<FishAudioSpeechModel, GenerationPortTable>> = {
+ "voice-design-1": table("voice-design-1"),
+ "voice-clone": table("voice-clone"),
+};
+
+export function sealFishAudioSpeechRequest(
+ model: FishAudioSpeechModel,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+): GenerationRequest {
+ return sealGenerationPortRequest(fishAudioSpeechPorts[model], ports);
+}
+
+export function sealFishAudioSpeechRequestDraft(
+ model: FishAudioSpeechModel,
+ ports: Readonly<Record<string, readonly GenerationPortValue[]>>,
+) {
+ return sealGenerationRequestDraft(fishAudioSpeechPorts[model], ports);
+}
+
+const base = defineExactModelModule({
+ module: fishAudioSpeechModuleRef,
+ endpoints: ([
+ ["voiceDesign", "voice-design-1", "FishAudioVoiceDesignRequest"],
+ ["voiceClone", "voice-clone", "FishAudioVoiceCloneRequest"],
+ ] as const).map(([key, model, requestTypeName]) => ({
+ key,
+ requestTypeName,
+ producerName: `request-${model}`,
+ ports: fishAudioSpeechPorts[model],
+ })),
+});
+
+export const fishAudioSpeechEndpoints = base.endpoints;
+
+const spokenAttributes: readonly SurfaceAttributeVocabulary[] = [
+ {
+ name: "id",
+ kind: "identifier",
+ required: true,
+ summary: "Names this speech request and prefixes the binding it publishes.",
+ },
+ {
+ name: "speech",
+ kind: "reference",
+ required: true,
+ summary: "The Text edge whose exact words are spoken.",
+ accepts: [textTypes.text],
+ },
+];
+
+const referencePort: readonly SurfacePortVocabulary[] = [{
+ name: "reference",
+ type: artifactTypes.blob,
+ summary: "The designed voice reference, addressed as `<id>.reference`.",
+}];
+
+const audioPort: readonly SurfacePortVocabulary[] = [{
+ name: "audio",
+ type: artifactTypes.blob,
+ summary: "The independent speech, addressed as `<id>.audio`.",
+}];
+
+export const fishAudioSpeechMarkupSurfaces = [
+ {
+ name: "voiceDesign", tag: "VoiceDesign", mode: "structured",
+ outputs: [fishAudioSpeechEndpoints.voiceDesign.draftType],
+ vocabulary: {
+ summary: "Creates a reusable voice reference from a natural-language voice description.",
+ attributes: spokenAttributes,
+ ports: referencePort,
+ text: "The element's own text is the voice description and is required.",
+ example: `<fish:VoiceDesign id="host" speech={story.segment.voiceSample.speech}>
+ A clear young woman with a grounded, confident conversational delivery.
+</fish:VoiceDesign>`,
+ notes: [
+ "The element accepts no child elements; only its text is read.",
+ "The result is an ordinary audio Resource that can be supplied anywhere an audio reference is accepted.",
+ ],
+ },
+ },
+ {
+ name: "voiceClone", tag: "VoiceClone", mode: "structured",
+ outputs: [
+ fishAudioSpeechEndpoints.voiceClone.draftType,
+ ...Object.values(fishAudioSpeechEndpoints.voiceClone.mediaBindings).map((binding) => binding.type),
+ ],
+ vocabulary: {
+ summary: "Creates independent speech in the voice heard in one accepted audio reference.",
+ attributes: [
+ ...spokenAttributes,
+ {
+ name: "voice",
+ kind: "reference",
+ required: true,
+ summary: "The audio Resource carrying the voice identity to reproduce.",
+ accepts: [artifactTypes.blob],
+ },
+ ],
+ ports: audioPort,
+ text: "The element's own text is an optional delivery instruction.",
+ example: `<fish:VoiceClone id="narration" speech={story.segment.reveal.speech} voice={host.reference}>
+ Quietly confident, with a short pause before the final word.
+</fish:VoiceClone>`,
+ notes: [
+ "The element accepts no child elements; only its text is read.",
+ "The voice reference remains a normal audio Resource rather than a separate identity record.",
+ ],
+ },
+ },
+] as const;
+
+export const fishAudioSpeechManifest = { ...base.manifest } as const;
+export const fishAudioSpeechComponent = base.component;
+export const fishAudioSpeechDefinition = { ...base, manifest: fishAudioSpeechManifest };
+
+export { createFishAudioSpeechAudioFragment } from "./fragment.js";
+export { decodeFishAudioVoiceCloneSurface, decodeFishAudioVoiceDesignSurface } from "./surface.js";
diff --git a/packages/fishaudio-speech/src/surface.ts b/packages/fishaudio-speech/src/surface.ts
new file mode 100644
index 00000000..67232642
--- /dev/null
+++ b/packages/fishaudio-speech/src/surface.ts
@@ -0,0 +1,193 @@
+import { artifactTypes } from "@hypit/artifact";
+import { generationPort, sealGenerationMediaBinding } from "@hypit/generation";
+import type { GenerationMediaPort, GenerationPortValue } from "@hypit/generation";
+import { exactModelMediaInputNames, exactModelTextInputName } from "@hypit/model-kit";
+import type { ExactModelEndpoint, ExactModelMediaInput, ExactModelTextInput } from "@hypit/model-kit";
+import type { CanonicalValue } from "@hypit/protocol";
+import { textTypes, verifyText } from "@hypit/text";
+import type {
+ MarkupAttributeValue,
+ StructuredElement,
+ StructuredSurfaceHandler,
+ SurfaceResolvedReference,
+} from "@hypit/markup";
+
+import { createFishAudioSpeechAudioFragment } from "./fragment.js";
+import { fishAudioSpeechEndpoints, sealFishAudioSpeechRequestDraft } from "./index.js";
+
+function sameType(left: SurfaceResolvedReference["type"], right: SurfaceResolvedReference["type"]): boolean {
+ return left.module.name === right.module.name && left.module.version === right.module.version && left.name === right.name;
+}
+
+function attributes(element: StructuredElement, required: readonly string[]): void {
+ const allowed = new Set(required);
+ const unknown = Object.keys(element.attributes).filter((name) => !allowed.has(name));
+ if (unknown.length > 0 || required.some((name) => element.attributes[name] === undefined)) {
+ throw new Error(`${element.name} requires ${required.join(", ")}`);
+ }
+}
+
+function stringAttribute(element: StructuredElement, name: string): string {
+ const value = element.attributes[name];
+ if (typeof value !== "string" || value.trim().length === 0) {
+ throw new Error(`${element.name}.${name} must be a non-empty string`);
+ }
+ return value.trim();
+}
+
+function referencePath(element: StructuredElement, name: string): string {
+ const value: MarkupAttributeValue | undefined = element.attributes[name];
+ if (typeof value !== "object" || value.kind !== "reference" || value.path.length === 0) {
+ throw new Error(`${element.name}.${name} must be a whole-value reference`);
+ }
+ return value.path;
+}
+
+function resolved(
+ element: StructuredElement,
+ name: string,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+): SurfaceResolvedReference {
+ const path = referencePath(element, name);
+ const value = resolveReference(path);
+ if (value === undefined) throw new Error(`${element.name}.${name} cannot resolve ${path}`);
+ return value;
+}
+
+function speechText(reference: SurfaceResolvedReference, subject: string): SurfaceResolvedReference {
+ if (!sameType(reference.type, textTypes.text)) throw new Error(`${subject} must reference Text`);
+ const value = reference.record?.value;
+ if (value !== undefined) {
+ if (value.kind !== "inline") throw new Error(`${subject} has an invalid authored Text value`);
+ verifyText(value.value);
+ }
+ return reference;
+}
+
+function voiceReference(reference: SurfaceResolvedReference, subject: string): SurfaceResolvedReference {
+ if (!sameType(reference.type, artifactTypes.blob)) throw new Error(`${subject} must reference an audio Resource`);
+ const value = reference.record?.value;
+ if (value !== undefined && (value.kind !== "blob" || !value.mediaType.startsWith("audio/"))) {
+ throw new Error(`${subject} must reference audio media`);
+ }
+ return reference;
+}
+
+function body(element: StructuredElement, required: boolean): string | undefined {
+ if (element.children.some((child) => child.kind === "element")) {
+ throw new Error(`${element.name} accepts instruction text only`);
+ }
+ const lines = element.children.map((child) => child.kind === "text" ? child.value : "").join("")
+ .replaceAll("\r\n", "\n").split("\n");
+ while (lines[0]?.trim() === "") lines.shift();
+ while (lines.at(-1)?.trim() === "") lines.pop();
+ const indents = lines.filter((line) => line.trim()).map((line) => /^\s*/u.exec(line)?.[0].length ?? 0);
+ const indent = indents.length === 0 ? 0 : Math.min(...indents);
+ const value = lines.map((line) => line.slice(indent).trimEnd()).join("\n").trim();
+ if (value.length === 0) {
+ if (required) throw new Error(`${element.name} requires a voice description in its body`);
+ return undefined;
+ }
+ return value;
+}
+
+function output(
+ element: StructuredElement,
+ endpoint: ExactModelEndpoint,
+ draft: ReturnType<typeof sealFishAudioSpeechRequestDraft>,
+ outputName: "reference" | "audio",
+ text: readonly { readonly input: ExactModelTextInput; readonly source: SurfaceResolvedReference }[],
+ media: readonly { readonly input: ExactModelMediaInput; readonly source: SurfaceResolvedReference }[] = [],
+) {
+ const id = stringAttribute(element, "id");
+ const draftId = `${id}.draft`;
+ const records: Array<{
+ readonly id: string;
+ readonly type: SurfaceResolvedReference["type"];
+ readonly value: { readonly kind: "inline"; readonly value: CanonicalValue };
+ readonly range: StructuredElement["range"];
+ }> = [{
+ id: draftId,
+ type: endpoint.draftType,
+ value: { kind: "inline", value: draft as unknown as CanonicalValue },
+ range: element.range,
+ }];
+ const inputs: Record<string, SurfaceResolvedReference["ref"] | { readonly kind: "record"; readonly id: string }> = {
+ draft: { kind: "record", id: draftId },
+ };
+ for (const item of text) inputs[exactModelTextInputName(item.input.name)] = item.source.ref;
+ for (const item of media) {
+ const binding = endpoint.mediaBindings[item.input.port];
+ if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${item.input.port}`);
+ const port = generationPort(endpoint.ports, item.input.port);
+ if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${item.input.port} is not media`);
+ const names = exactModelMediaInputNames(item.input.name);
+ const bindingId = `${id}.${item.input.name}.binding`;
+ records.push({
+ id: bindingId,
+ type: binding.type,
+ value: {
+ kind: "inline",
+ value: sealGenerationMediaBinding(port as GenerationMediaPort, { role: "audio" }) as unknown as CanonicalValue,
+ },
+ range: element.range,
+ });
+ inputs[names.binding] = { kind: "record", id: bindingId };
+ inputs[names.artifact] = item.source.ref;
+ }
+ const fragment = createFishAudioSpeechAudioFragment(
+ endpoint,
+ media.map((item) => item.input),
+ text.map((item) => item.input),
+ );
+ return {
+ records,
+ components: [{
+ id,
+ fragment: fragment.id,
+ inputs,
+ outputs: { audio: `${id}.${outputName}` },
+ range: element.range,
+ }],
+ fragments: [fragment],
+ };
+}
+
+function speechInput(
+ element: StructuredElement,
+ resolveReference: (path: string) => SurfaceResolvedReference | undefined,
+) {
+ return {
+ input: { name: "speech", port: "text" },
+ source: speechText(resolved(element, "speech", resolveReference), `${element.name}.speech`),
+ } as const;
+}
+
+export const decodeFishAudioVoiceDesignSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ attributes(element, ["id", "speech"]);
+ return output(
+ element,
+ fishAudioSpeechEndpoints.voiceDesign,
+ sealFishAudioSpeechRequestDraft("voice-design-1", { voiceDescription: [body(element, true)!] }),
+ "reference",
+ [speechInput(element, resolveReference)],
+ );
+};
+
+export const decodeFishAudioVoiceCloneSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => {
+ attributes(element, ["id", "speech", "voice"]);
+ const instruction = body(element, false);
+ const ports: Record<string, readonly GenerationPortValue[]> = {};
+ if (instruction !== undefined) ports.instruction = [instruction];
+ return output(
+ element,
+ fishAudioSpeechEndpoints.voiceClone,
+ sealFishAudioSpeechRequestDraft("voice-clone", ports),
+ "audio",
+ [speechInput(element, resolveReference)],
+ [{
+ input: { name: "voice", port: "voiceReference" },
+ source: voiceReference(resolved(element, "voice", resolveReference), `${element.name}.voice`),
+ }],
+ );
+};
diff --git a/packages/fishaudio-speech/test/fishaudio-speech.test.ts b/packages/fishaudio-speech/test/fishaudio-speech.test.ts
new file mode 100644
index 00000000..08cc75df
--- /dev/null
+++ b/packages/fishaudio-speech/test/fishaudio-speech.test.ts
@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { artifactTypes } from "@hypit/artifact";
+import { generationTypes } from "@hypit/generation";
+import { parseStructuredElement } from "@hypit/markup";
+import type { SurfaceResolvedReference } from "@hypit/markup";
+import { sealText, textTypes } from "@hypit/text";
+
+import fishAudioNodePackage from "../src/activation.js";
+import {
+ decodeFishAudioVoiceCloneSurface,
+ decodeFishAudioVoiceDesignSurface,
+ fishAudioSpeechEndpoints,
+ fishAudioSpeechMarkupSurfaces,
+ fishAudioSpeechModels,
+ fishAudioSpeechPorts,
+ sealFishAudioSpeechRequest,
+} from "../src/index.js";
+
+function parsed(source: string) {
+ return parseStructuredElement({ name: "fish.svml", text: source }, 0).element;
+}
+
+const speech = sealText("Exact authored words stay exact.");
+const speechReference: SurfaceResolvedReference = {
+ path: "story.segment.opening.speech",
+ ref: { kind: "record", id: "story.segment.opening.speech" },
+ type: textTypes.text,
+ record: {
+ id: "story.segment.opening.speech",
+ type: textTypes.text,
+ value: { kind: "inline", value: speech },
+ },
+};
+const voiceReference: SurfaceResolvedReference = {
+ path: "host.reference",
+ ref: { kind: "record", id: "host.reference" },
+ type: artifactTypes.blob,
+ record: {
+ id: "host.reference",
+ type: artifactTypes.blob,
+ value: { kind: "blob", resource: "res_voice-reference", size: 4, mediaType: "audio/wav" },
+ },
+};
+
+const context = (source: string, voice: SurfaceResolvedReference = voiceReference) => ({
+ sourceName: "fish.svml",
+ element: parsed(source),
+ resolveReference: (path: string) => path === speechReference.path
+ ? speechReference : path === voice.path ? voice : undefined,
+ resolveAsset: () => { throw new Error("no asset"); },
+});
+
+test("Fish Audio Speech declares the two exact audio request shapes", () => {
+ assert.deepEqual(fishAudioSpeechModels, ["voice-design-1", "voice-clone"]);
+ assert.ok(Object.values(fishAudioSpeechPorts).every((ports) => ports.result === "audio"));
+ assert.ok(Object.values(fishAudioSpeechEndpoints).every((endpoint) => endpoint.returns.name === generationTypes.audioSet.name));
+ assert.throws(() => sealFishAudioSpeechRequest("voice-design-1", {
+ text: ["Do not rewrite me."],
+ }), /voiceDescription is required/u);
+ assert.throws(() => sealFishAudioSpeechRequest("voice-clone", {
+ text: ["Do not rewrite me."],
+ }), /voiceReference is required/u);
+});
+
+test("the installed author package contributes the two explicit speech operations", () => {
+ assert.equal(fishAudioNodePackage.format, "hypit.node-package@1");
+ assert.equal(fishAudioNodePackage.modules[0]?.manifest.version, "1");
+ assert.deepEqual(fishAudioSpeechMarkupSurfaces.map((surface) => surface.name), ["voiceDesign", "voiceClone"]);
+});
+
+test("VoiceDesign publishes one reusable voice reference", async () => {
+ const result = await decodeFishAudioVoiceDesignSurface(context(`<fish:VoiceDesign id="host" speech={story.segment.opening.speech}>
+ A clear, confident young woman with a grounded conversational tone.
+ </fish:VoiceDesign>`));
+ assert.deepEqual(result.components[0]!.outputs, { audio: "host.reference" });
+ const exported = result.fragments[0]!.exports.find((item) => item.name === "audio");
+ assert.deepEqual(exported?.type, artifactTypes.blob);
+ assert.deepEqual(result.components[0]!.inputs["speech:text"], {
+ kind: "record",
+ id: "story.segment.opening.speech",
+ });
+ const request = result.records[0]!.value.kind === "inline"
+ ? result.records[0]!.value.value as Record<string, unknown> : {};
+ assert.equal((request.ports as Record<string, unknown[]>).text, undefined);
+});
+
+test("VoiceClone binds an accepted audio Resource and publishes independent speech", async () => {
+ const result = await decodeFishAudioVoiceCloneSurface(context(`<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}>
+ Calm and restrained.
+ </fish:VoiceClone>`));
+ assert.deepEqual(result.components[0]!.outputs, { audio: "narration.audio" });
+ assert.deepEqual(result.components[0]!.inputs["voice:artifact"], { kind: "record", id: "host.reference" });
+ assert.ok(result.components[0]!.inputs["voice:binding"] !== undefined);
+});
+
+test("VoiceClone refuses a known non-audio Resource", () => {
+ const image: SurfaceResolvedReference = {
+ ...voiceReference,
+ record: {
+ ...voiceReference.record!,
+ value: { kind: "blob", resource: "res_image", size: 4, mediaType: "image/png" },
+ },
+ };
+ assert.throws(() => decodeFishAudioVoiceCloneSurface(context(
+ `<fish:VoiceClone id="narration" speech={story.segment.opening.speech} voice={host.reference}/>` ,
+ image,
+ )), /must reference audio media/u);
+});
diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.md
index 23b0538e..f0a30fa4 100644
--- a/packages/provider-hypihub/README.md
+++ b/packages/provider-hypihub/README.md
@@ -91,13 +91,23 @@ catalog to verify configured capabilities; ordinary preflight never makes that r
declares HypiHub's public pricing page, `https://hypit.ai/commercial/pricing/`, as its price source.
For each selected Need, `readPricing` resolves the corresponding HypiHub model and returns the service's
authenticated `GET /v1/pricing?model=<model>` response unchanged together with that URL. It covers
-generation, alignment, Voice Design, and Voice Clone through the same mechanism;
+generation, alignment, Voice Design and Voice Clone through the same mechanism;
the Provider does not maintain a second list of billing formulas or calculate a request total.
-HypiHub declares MiMo Voice Design and Voice Clone together with every other capability it serves; it
-never hides one. Voice Design produces an accepted voice-reference Resource, and Voice Clone uses
-that reference to produce independent speech. Hypit does not expose MiMo preset voices. When another
-selected Endpoint offers the same capability (a local WhisperX or the official MiMo
+HypiHub declares every speech capability it serves together with every other capability; it never
+hides one. Voice Design produces an accepted voice-reference Resource, and Voice Clone uses that
+reference to produce independent speech. All of them use `POST /v1/audio/speech`:
+
+| Package | Capability | HypiHub model | Request fields |
+| --- | --- | --- | --- |
+| `@hypit/mimo-speech` | `mimo-v2.5-tts-voicedesign` | `mimo-v2.5-tts-voicedesign` | `input`, `voice_description` |
+| `@hypit/mimo-speech` | `mimo-v2.5-tts-voiceclone` | `mimo-v2.5-tts-voiceclone` | `input`, `reference_audio`, optional `prompt` |
+| `@hypit/fishaudio-speech` | `voice-design-1` | `fishaudio/voice-design-1` | `input`, `voice_description` |
+| `@hypit/fishaudio-speech` | `voice-clone` | `fishaudio/voice-clone` | `input`, `reference_audio`, optional `prompt`, constant `voice_description` title |
+| `@hypit/elevenlabs-speech` | `eleven_ttv_v3` | `eleven_ttv_v3` | `input`, `voice_description` |
+
+Each returned preview becomes one member of the audio set. Hypit does not expose preset voices. When
+another selected Endpoint offers the same capability (a local WhisperX or the official MiMo
Provider), the Runtime Profile's `bindings` say which Endpoint serves it.
Execution policy remains local to this Provider:
diff --git a/packages/provider-hypihub/package.json b/packages/provider-hypihub/package.json
index d11ca557..dbbdb191 100644
--- a/packages/provider-hypihub/package.json
+++ b/packages/provider-hypihub/package.json
@@ -18,6 +18,8 @@
},
"devDependencies": {
"@hypit/driver-node": "workspace:*",
+ "@hypit/elevenlabs-speech": "workspace:*",
+ "@hypit/fishaudio-speech": "workspace:*",
"@hypit/mimo-speech": "workspace:*",
"@hypit/seedance": "workspace:*",
"@hypit/speech": "workspace:*",
diff --git a/packages/provider-hypihub/src/mapping.ts b/packages/provider-hypihub/src/mapping.ts
index 8a5b64e6..eb205215 100644
--- a/packages/provider-hypihub/src/mapping.ts
+++ b/packages/provider-hypihub/src/mapping.ts
@@ -9,6 +9,8 @@ const SEEDREAM: ModuleRef = { name: "@hypit/seedream", version: "1" };
const MINIMAX: ModuleRef = { name: "@hypit/minimax-h3", version: "1" };
const GROK: ModuleRef = { name: "@hypit/grok-imagine", version: "1" };
const MIMO_SPEECH: ModuleRef = { name: "@hypit/mimo-speech", version: "1" };
+const FISHAUDIO_SPEECH: ModuleRef = { name: "@hypit/fishaudio-speech", version: "1" };
+const ELEVENLABS_SPEECH: ModuleRef = { name: "@hypit/elevenlabs-speech", version: "1" };
const seedance = (name: string, model: string): GenerationWireMapping => ({
capability: { module: SEEDANCE, name }, result: "video", routes: [{ model }],
@@ -132,6 +134,30 @@ export const hypiHubMappings: readonly GenerationWireMapping[] = [
voiceReference: { as: "urlArray", field: "reference_audio" },
},
},
+ {
+ capability: { module: FISHAUDIO_SPEECH, name: "voice-design-1" }, result: "audio", routes: [{ model: "fishaudio/voice-design-1" }],
+ fields: {
+ text: { as: "value", field: "input" },
+ voiceDescription: { as: "value", field: "voice_description" },
+ },
+ },
+ {
+ capability: { module: FISHAUDIO_SPEECH, name: "voice-clone" }, result: "audio", routes: [{ model: "fishaudio/voice-clone" }],
+ // Fish Audio titles its transient cloned voice; the title has no authored meaning.
+ constants: { voice_description: "reference" },
+ fields: {
+ text: { as: "value", field: "input" },
+ instruction: { as: "value", field: "prompt" },
+ voiceReference: { as: "urlArray", field: "reference_audio" },
+ },
+ },
+ {
+ capability: { module: ELEVENLABS_SPEECH, name: "eleven_ttv_v3" }, result: "audio", routes: [{ model: "eleven_ttv_v3" }],
+ fields: {
+ text: { as: "value", field: "input" },
+ voiceDescription: { as: "value", field: "voice_description" },
+ },
+ },
];
function capabilityKey(ref: CapabilityRef): string {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ee03080d..cc4a30d6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -93,6 +93,9 @@ importers:
'@hypit/deck-track-studio':
specifier: workspace:*
version: link:packages/deck-track-studio
+ '@hypit/elevenlabs-speech':
+ specifier: workspace:*
+ version: link:packages/elevenlabs-speech
'@hypit/estimate':
specifier: workspace:*
version: link:packages/estimate
@@ -102,6 +105,9 @@ importers:
'@hypit/film-studio':
specifier: workspace:*
version: link:packages/film-studio
+ '@hypit/fishaudio-speech':
+ specifier: workspace:*
+ version: link:packages/fishaudio-speech
'@hypit/fonts-open':
specifier: workspace:*
version: link:packages/fonts-open
@@ -970,6 +976,27 @@ importers:
specifier: workspace:*
version: link:../source
+ packages/elevenlabs-speech:
+ dependencies:
+ '@hypit/artifact':
+ specifier: workspace:*
+ version: link:../artifact
+ '@hypit/generation':
+ specifier: workspace:*
+ version: link:../generation
+ '@hypit/markup':
+ specifier: workspace:*
+ version: link:../markup
+ '@hypit/model-kit':
+ specifier: workspace:*
+ version: link:../model-kit
+ '@hypit/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@hypit/text':
+ specifier: workspace:*
+ version: link:../text
+
packages/endpoint-kit:
dependencies:
'@hypit/protocol':
@@ -1070,6 +1097,27 @@ importers:
specifier: workspace:*
version: link:../studio-adapter
+ packages/fishaudio-speech:
+ dependencies:
+ '@hypit/artifact':
+ specifier: workspace:*
+ version: link:../artifact
+ '@hypit/generation':
+ specifier: workspace:*
+ version: link:../generation
+ '@hypit/markup':
+ specifier: workspace:*
+ version: link:../markup
+ '@hypit/model-kit':
+ specifier: workspace:*
+ version: link:../model-kit
+ '@hypit/protocol':
+ specifier: workspace:*
+ version: link:../protocol
+ '@hypit/text':
+ specifier: workspace:*
+ version: link:../text
+
packages/fonts-open:
dependencies:
'@hypit/markup':
@@ -2065,6 +2113,12 @@ importers:
'@hypit/driver-node':
specifier: workspace:*
version: link:../driver-node
+ '@hypit/elevenlabs-speech':
+ specifier: workspace:*
+ version: link:../elevenlabs-speech
+ '@hypit/fishaudio-speech':
+ specifier: workspace:*
+ version: link:../fishaudio-speech
'@hypit/mimo-speech':
specifier: workspace:*
version: link:../mimo-speech
diff --git a/skills/hypit/references/playbooks/craft/voice-and-performance.md b/skills/hypit/references/playbooks/craft/voice-and-performance.md
index 71d72d61..61074b95 100644
--- a/skills/hypit/references/playbooks/craft/voice-and-performance.md
+++ b/skills/hypit/references/playbooks/craft/voice-and-performance.md
@@ -130,9 +130,12 @@ B-roll or graphics happen to hide a visible performance. Use Voice Design to est
reusable voice and Voice Clone to perform the actual Segment; the casting sample and the finished
delivery have different jobs.
-Use `hypit vocabulary @hypit/mimo-speech` for the installed Voice Design and Voice Clone author
-Surfaces. The Source chooses those model semantics; the Runtime Profile independently chooses the
-Endpoint that can perform them.
+Each vendor's speech models are one package with its own Source import: `@hypit/mimo-speech`
+(Xiaomi MiMo Voice Design and Voice Clone), `@hypit/fishaudio-speech` (Fish Audio Voice Design and
+Voice Clone) and `@hypit/elevenlabs-speech` (ElevenLabs Voice Design). Use `hypit vocabulary` with a
+package name for its author Surfaces. A voice reference designed by one package is an ordinary audio
+Resource that another package's Voice Clone accepts. The Source chooses those model semantics; the
+Runtime Profile independently chooses the Endpoint that can perform them.
## Treat narration as a sound-picture relationship