Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
feat(video-cli): add observe, transcribe and speak creation tools
kashorin48ed5d7355c1
diff --git a/packages/video-cli/README.md b/packages/video-cli/README.mdindex 5e174227..63d6e395 100644--- a/packages/video-cli/README.md+++ b/packages/video-cli/README.md@@ -29,6 +29,22 @@ hypit cancel <build-id>hypit doctor```+Creation-time tools run one immediate request through the selected Runtime Profile, with no Build,+Result or state. Each names the Endpoint and its price page before it runs, and writes exactly one+file the caller chose:++```bash+hypit observe <media…> --instruction <text|file> --prompt <text|file> --to notes/frame.md+hypit transcribe reference.mp4 --to notes/reference.transcript.json [--language en|zh|es]+hypit speak --text line.txt --voice "warm, unhurried" --to voice/intro.mp3+```++`observe` uses the Profile's Gemini Endpoint, `transcribe` its `whisperx-alignment` Endpoint (after+extracting 16 kHz mono speech audio with ffmpeg), `speak` its MiMo VoiceDesign Endpoint and reports+the audio's duration so the author can write it as a literal. `--runtime <profile>` names the Profile;+otherwise the project's `hypit runtime use` selection is read. Slow paid generation — pictures and+clips — is not a creation-time tool: declare it in the Source and go through `plan` and `build`.+Install the `hypit` Distribution globally once. It resolves its own TypeScript loader and CLI, so itneither invokes npm per command nor requires a project to contain Hypit's `package.json`.diff --git a/packages/video-cli/package.json b/packages/video-cli/package.jsonindex f75fd3c5..a92b540e 100644--- a/packages/video-cli/package.json+++ b/packages/video-cli/package.json@@ -12,9 +12,19 @@"@hypit/cli": "workspace:*","@hypit/compiler-markup-node": "workspace:*","@hypit/credential-store-os": "workspace:*",+ "@hypit/driver-node": "workspace:*",+ "@hypit/gemini": "workspace:*",+ "@hypit/generation": "workspace:*","@hypit/markup": "workspace:*",+ "@hypit/mimo-tts": "workspace:*","@hypit/package-loader-node": "workspace:*",+ "@hypit/protocol": "workspace:*","@hypit/provider-hypihub": "workspace:*",+ "@hypit/runtime": "workspace:*",+ "@hypit/speech": "workspace:*",+ "@hypit/speech-evidence": "workspace:*",+ "@hypit/text": "workspace:*",+ "@hypit/whisperx": "workspace:*","@hypit/runtime-host-node": "workspace:*","@hypit/runtime-local": "workspace:*","@hypit/workspace-fs-node": "workspace:*"diff --git a/packages/video-cli/src/cli.ts b/packages/video-cli/src/cli.tsindex d1879461..0d49f564 100644--- a/packages/video-cli/src/cli.ts+++ b/packages/video-cli/src/cli.ts@@ -1,6 +1,7 @@#!/usr/bin/env nodeimport { renderCliError, writeCliHelp } from "@hypit/cli";import type { CliIo } from "@hypit/cli";+import { creationCommands, isCreationCommand, writeCreationHelp } from "./creation.js";const argv = process.argv.slice(2);const json = argv.includes("--json");@@ -59,7 +60,15 @@ const io: CliIo = {async function main(): Promise<void> {if (argv.length === 0 || argv[0] === "help" || argv.includes("--help")) {const topic = argv[0] === "help" ? argv[1] : argv.includes("--help") ? argv[0] : undefined;+ if (isCreationCommand(topic)) {+ writeCreationHelp(io, topic);+ return;+ }writeCliHelp(io, topic);+ if (topic === undefined) {+ io.write(`\nCreation tools (one request through the selected Runtime Profile, no Build)\n${+ creationCommands.map((item) => ` ${item}`).join("\n")}\n hypit help <tool> for each\n`);+ }return;}const { runVideoCli } = await import("./index.js");diff --git a/packages/video-cli/src/creation.ts b/packages/video-cli/src/creation.tsnew file mode 100644index 00000000..5ea45ac0--- /dev/null+++ b/packages/video-cli/src/creation.ts@@ -0,0 +1,518 @@+import { spawn } from "node:child_process";+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { dirname, extname, join, resolve } from "node:path";++import { findRuntimeProfile } from "@hypit/cli";+import type { CliIo } from "@hypit/cli";+import { MemoryResourceStore } from "@hypit/driver-node";+import { geminiCapabilities, geminiModels, sealGeminiRequest } from "@hypit/gemini";+import type { GeminiMediaPart, GeminiModel } from "@hypit/gemini";+import { verifyGeneratedAudioSet } from "@hypit/generation";+import type { GeneratedAudioSet } from "@hypit/generation";+import { mimoTtsEndpoints, sealMimoTtsRequest } from "@hypit/mimo-tts";+import type { CanonicalValue, CapabilityRef, Need, StoredValue } from "@hypit/protocol";+import type { ResourceStore } from "@hypit/runtime";+import type { RuntimeHostCapabilityProvider } from "@hypit/runtime-host-node";+import { sealSpeechEvidenceAudio } from "@hypit/speech";+import { speechEvidenceTypes } from "@hypit/speech-evidence";+import type { AlignedTranscriptEvidence } from "@hypit/speech-evidence";+import { textTypes } from "@hypit/text";+import { whisperXCapabilities, whisperXRequestForEvidenceAudio } from "@hypit/whisperx";+import type { WhisperXLanguage } from "@hypit/whisperx";++import { videoCliDistribution } from "./distribution.js";++/**+ * Creation-time tools: see a picture, hear a recording, speak a line.+ *+ * Each is one immediate Need executed through the selected Runtime Profile, exactly as a Build would+ * resolve it — same Endpoint, same credential, same Provider — without a Build, Result or state. The+ * command names the Endpoint and its price page before it spends anything, writes one file the caller+ * chose, and nothing else. Slow paid generation (pictures, clips) is not here: it is a Build.+ */++export const creationCommands = ["observe", "transcribe", "speak"] as const;+export type CreationCommand = typeof creationCommands[number];++/** The slice of the Runtime host these commands use; tests hand in a fake. */+export type CreationHost = {+ providers(capabilities: readonly CapabilityRef[]): Promise<readonly RuntimeHostCapabilityProvider[]>;+ invoke(need: Need, resources: ResourceStore): Promise<{ readonly value: StoredValue }>;+};++export type CreationEnvironment = {+ /** Open the host behind `--runtime`, or behind the project's own selection when none is named. */+ readonly openHost: (runtime: string | undefined) => Promise<{ readonly profile: string; readonly host: CreationHost }>;+ readonly cwd: string;+};++function assert(condition: unknown, message: string): asserts condition {+ if (!condition) throw new Error(message);+}++export function isCreationCommand(value: string | undefined): value is CreationCommand {+ return (creationCommands as readonly string[]).includes(value ?? "");+}++async function nearestPackageRoot(start: string): Promise<string> {+ let directory = resolve(start);+ while (true) {+ const manifest = resolve(directory, "package.json");+ if (await stat(manifest).then((item) => item.isFile(), () => false)) return directory;+ const parent = dirname(directory);+ if (parent === directory) return resolve(start);+ directory = parent;+ }+}++/** The default environment: the project's Profile, opened through this Distribution's Runtime. */+export function creationEnvironment(cwd = process.cwd()): CreationEnvironment {+ return {+ cwd,+ openHost: async (runtime) => {+ let profile: string;+ if (runtime !== undefined) {+ profile = resolve(cwd, runtime);+ } else {+ const projectRoot = await nearestPackageRoot(cwd);+ const selected = await findRuntimeProfile(projectRoot);+ assert(selected !== undefined,+ `No Runtime Profile is selected for ${projectRoot}; run hypit runtime use <profile> there, or pass --runtime <profile>`);+ profile = selected.profile;+ }+ const host = await videoCliDistribution.openRuntimeHost(profile, {+ packageRoot: await nearestPackageRoot(dirname(profile)),+ ...(videoCliDistribution.packageRoot === undefined ? {} : { distributionPackageRoot: videoCliDistribution.packageRoot }),+ });+ return { profile, host };+ },+ };+}++// ---------------------------------------------------------------------------------------------------+// Arguments++type Parsed = {+ readonly positionals: readonly string[];+ readonly options: ReadonlyMap<string, string>;+ readonly json: boolean;+};++function parseArguments(argv: readonly string[], allowed: readonly string[]): Parsed {+ const positionals: string[] = [];+ const options = new Map<string, string>();+ let json = false;+ for (let index = 1; index < argv.length; index += 1) {+ const item = argv[index]!;+ if (item === "--json") { json = true; continue; }+ if (item === "--debug" || item === "--verbose" || item === "--no-color") continue;+ if (item === "--color") { index += 1; continue; }+ if (!item.startsWith("--")) { positionals.push(item); continue; }+ if (!allowed.includes(item)) throw new Error(`unknown option ${item}`);+ if (options.has(item)) throw new Error(`${item} cannot be repeated`);+ const value = argv[index + 1];+ if (value === undefined || value.startsWith("--")) throw new Error(`${item} requires a value`);+ options.set(item, value);+ index += 1;+ }+ return { positionals, options, json };+}++function required(parsed: Parsed, option: string, hint: string): string {+ const value = parsed.options.get(option);+ if (value === undefined) throw new Error(`${option} is required: ${hint}`);+ return value;+}++/** An option that takes prose: the text itself, or the path of a file holding it. */+async function textOrFile(value: string, option: string, cwd: string): Promise<string> {+ const path = resolve(cwd, value);+ const isFile = await stat(path).then((item) => item.isFile(), () => false);+ const text = (isFile ? await readFile(path, "utf8") : value).trim();+ if (text.length === 0) throw new Error(isFile ? `${option} file ${path} is empty` : `${option} is empty`);+ return text;+}++async function destination(parsed: Parsed, cwd: string): Promise<string> {+ const to = resolve(cwd, required(parsed, "--to", "the file to write"));+ if (await stat(to).then(() => true, () => false)) throw new Error(`Destination ${to} already exists`);+ return to;+}++async function writeNew(path: string, content: string | Uint8Array): Promise<void> {+ await mkdir(dirname(path), { recursive: true });+ try {+ await writeFile(path, content, { flag: "wx" });+ } catch (error) {+ if (error instanceof Error && "code" in error && error.code === "EEXIST") {+ throw new Error(`Destination ${path} already exists`);+ }+ throw error;+ }+}++// ---------------------------------------------------------------------------------------------------+// Provider and price page++function capabilityName(capability: CapabilityRef): string {+ return `${capability.module.name}@${capability.module.version}#${capability.name}`;+}++/** Which Endpoint will serve this call and where its Provider publishes prices, before spending. */+async function selectedProvider(host: CreationHost, capability: CapabilityRef, profile: string): Promise<RuntimeHostCapabilityProvider> {+ const [provider] = await host.providers([capability]);+ const subject = capabilityName(capability);+ assert(provider !== undefined && provider.status !== "unresolved",+ `No Endpoint in ${profile} serves ${subject}; hypit plan --runtime ${profile} shows which Endpoint each capability needs`);+ assert(provider.status !== "ambiguous",+ `Several Endpoints in ${profile} serve ${subject}: ${(provider.endpoints ?? []).join(", ")}; keep exactly one`);+ return provider;+}++function priceLine(provider: RuntimeHostCapabilityProvider): string {+ return provider.pricing === undefined+ ? "price source unknown"+ : provider.pricing.kind === "local" ? "local, no Provider charge" : provider.pricing.url;+}++function providerView(provider: RuntimeHostCapabilityProvider) {+ return {+ endpoint: provider.endpoint ?? null,+ use: provider.use ?? null,+ pricing: provider.pricing ?? null,+ };+}++function providerLine(provider: RuntimeHostCapabilityProvider): string {+ return `${provider.endpoint} (${provider.use}) · ${priceLine(provider)}`;+}++// ---------------------------------------------------------------------------------------------------+// Media++const MEDIA_TYPES: Readonly<Record<string, string>> = {+ ".png": "image/png",+ ".jpg": "image/jpeg",+ ".jpeg": "image/jpeg",+ ".webp": "image/webp",+ ".gif": "image/gif",+ ".mp4": "video/mp4",+ ".m4v": "video/mp4",+ ".mov": "video/quicktime",+ ".webm": "video/webm",+ ".wav": "audio/wav",+ ".mp3": "audio/mpeg",+ ".m4a": "audio/mp4",+ ".flac": "audio/flac",+ ".ogg": "audio/ogg",+};++function mediaType(path: string): string {+ const type = MEDIA_TYPES[extname(path).toLowerCase()];+ assert(type !== undefined, `${path} has no media type this command knows; supported: ${Object.keys(MEDIA_TYPES).join(", ")}`);+ return type;+}++function run(executable: string, args: readonly string[]): Promise<string> {+ return new Promise((resolveRun, reject) => {+ const child = spawn(executable, args, { stdio: ["ignore", "pipe", "pipe"] });+ const out: Buffer[] = [];+ const err: Buffer[] = [];+ child.stdout.on("data", (chunk: Buffer) => out.push(chunk));+ child.stderr.on("data", (chunk: Buffer) => err.push(chunk));+ child.on("error", (error) => reject(new Error(`${executable} could not start: ${error.message}`)));+ child.on("close", (code) => {+ if (code === 0) resolveRun(Buffer.concat(out).toString("utf8"));+ else reject(new Error(`${executable} exited with ${code}: ${Buffer.concat(err).toString("utf8").trim()}`));+ });+ });+}++const EVIDENCE_SAMPLE_RATE = 16_000;++type WavShape = { readonly sampleRate: number; readonly channels: number; readonly bits: number; readonly codec: number; readonly dataBytes: number };++/** Read a RIFF WAV header; undefined when the bytes are not a WAV file. */+function wavShape(bytes: Uint8Array): WavShape | undefined {+ const fourCc = (offset: number) => String.fromCharCode(...bytes.subarray(offset, offset + 4));+ if (bytes.byteLength < 44 || fourCc(0) !== "RIFF" || fourCc(8) !== "WAVE") return undefined;+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);+ let offset = 12;+ let format: Omit<WavShape, "dataBytes"> | undefined;+ let dataBytes: number | undefined;+ while (offset + 8 <= bytes.byteLength) {+ const name = fourCc(offset);+ const size = view.getUint32(offset + 4, true);+ const body = offset + 8;+ if (body + size > bytes.byteLength) return undefined;+ if (name === "fmt " && size >= 16) {+ format = { codec: view.getUint16(body, true), channels: view.getUint16(body + 2, true), sampleRate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) };+ } else if (name === "data") {+ dataBytes = size;+ }+ offset = body + size + (size % 2);+ }+ return format === undefined || dataBytes === undefined ? undefined : { ...format, dataBytes };+}++function canonicalEvidence(shape: WavShape | undefined): shape is WavShape {+ return shape !== undefined && shape.codec === 1 && shape.channels === 1 && shape.sampleRate === EVIDENCE_SAMPLE_RATE && shape.bits === 16;+}++/** The 16 kHz mono PCM WAV WhisperX measures; extracted with ffmpeg unless the file already is one. */+async function speechEvidenceBytes(path: string): Promise<{ readonly bytes: Uint8Array; readonly sampleFrames: number; readonly extracted: boolean }> {+ const original = new Uint8Array(await readFile(path));+ const shape = wavShape(original);+ if (canonicalEvidence(shape)) return { bytes: original, sampleFrames: shape.dataBytes / 2, extracted: false };+ const scratch = await mkdtemp(join(tmpdir(), "hypit-transcribe-"));+ try {+ const target = join(scratch, "speech.wav");+ await run("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-i", path, "-map", "0:a:0", "-vn", "-ac", "1", "-ar", String(EVIDENCE_SAMPLE_RATE), "-c:a", "pcm_s16le", "-bitexact", target]);+ const bytes = new Uint8Array(await readFile(target));+ const extracted = wavShape(bytes);+ assert(canonicalEvidence(extracted), "ffmpeg did not produce 16 kHz mono PCM audio");+ return { bytes, sampleFrames: extracted.dataBytes / 2, extracted: true };+ } finally {+ await rm(scratch, { recursive: true, force: true });+ }+}++/** Seconds of audio in a file: read from a WAV header, otherwise asked of ffprobe when it is present. */+async function audioSeconds(path: string, bytes: Uint8Array): Promise<number | undefined> {+ const shape = wavShape(bytes);+ if (shape !== undefined && shape.codec === 1 && shape.channels > 0 && shape.sampleRate > 0 && shape.bits > 0) {+ return round(shape.dataBytes / (shape.channels * (shape.bits / 8)) / shape.sampleRate);+ }+ try {+ const seconds = Number((await run("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path])).trim());+ return Number.isFinite(seconds) && seconds > 0 ? round(seconds) : undefined;+ } catch {+ return undefined;+ }+}++function round(value: number): number { return Number(value.toFixed(3)); }+function seconds(sample: number | undefined): number | undefined {+ return sample === undefined ? undefined : round(sample / EVIDENCE_SAMPLE_RATE);+}++// ---------------------------------------------------------------------------------------------------+// observe++async function observe(argv: readonly string[], io: CliIo, environment: CreationEnvironment): Promise<void> {+ const parsed = parseArguments(argv, ["--instruction", "--prompt", "--model", "--to", "--runtime"]);+ assert(parsed.positionals.length > 0, "observe requires at least one media file to look at");+ const modelName = parsed.options.get("--model") ?? geminiModels[0];+ assert((geminiModels as readonly string[]).includes(modelName),+ `Gemini model ${modelName} is not declared by @hypit/gemini; declared models: ${geminiModels.join(", ")}`);+ const model = modelName as GeminiModel;+ const instruction = await textOrFile(required(parsed, "--instruction", "who the observer is and what it returns"), "--instruction", environment.cwd);+ const prompt = await textOrFile(required(parsed, "--prompt", "the question to answer about the media"), "--prompt", environment.cwd);+ const to = await destination(parsed, environment.cwd);+ const { profile, host } = await environment.openHost(parsed.options.get("--runtime"));+ const capability = geminiCapabilities[model];+ const provider = await selectedProvider(host, capability, profile);+ if (!parsed.json) io.write(`Observing with ${model} through ${providerLine(provider)}\n`);+ const resources = new MemoryResourceStore();+ const media: GeminiMediaPart[] = [];+ for (const item of parsed.positionals) {+ const path = resolve(environment.cwd, item);+ media.push({ artifact: await resources.put(new Uint8Array(await readFile(path)), mediaType(path)) });+ }+ const need: Need = {+ id: "need:hypit-observe",+ capability,+ returns: textTypes.text,+ constraints: sealGeminiRequest({ instruction, prompt, media }) as unknown as CanonicalValue,+ result: "record:hypit-observe",+ };+ const fulfillment = await host.invoke(need, resources);+ assert(fulfillment.value.kind === "inline", "the observation came back by reference");+ const text = (fulfillment.value.value as { readonly value?: unknown } | null)?.value;+ assert(typeof text === "string", "the observation is not Text");+ await writeNew(to, `${text}\n`);+ const view = {+ format: "hypit.video-cli-observe@1",+ model,+ media: parsed.positionals.map((item) => resolve(environment.cwd, item)),+ ...providerView(provider),+ characters: text.length,+ path: to,+ };+ if (parsed.json) io.write(`${JSON.stringify(view, null, 2)}\n`);+ else io.write(`✓ Observation written\n\n ${text.length} characters\n ${to}\n`);+}++// ---------------------------------------------------------------------------------------------------+// transcribe++type TranscriptWord = { readonly text: string; readonly start_seconds?: number; readonly end_seconds?: number; readonly score?: number };+type TranscriptPassage = { readonly text: string; readonly start_seconds?: number; readonly end_seconds?: number; readonly words: readonly TranscriptWord[] };++function passagesInSeconds(aligned: AlignedTranscriptEvidence): readonly TranscriptPassage[] {+ return aligned.passages.map((passage) => {+ const words = passage.words.map((word): TranscriptWord => {+ const start = seconds(word.startSample);+ const end = seconds(word.endSampleExclusive);+ return {+ text: word.text,+ ...(start === undefined ? {} : { start_seconds: start }),+ ...(end === undefined ? {} : { end_seconds: end }),+ ...(word.score === undefined ? {} : { score: word.score }),+ };+ });+ const start = seconds(passage.startSample);+ const end = seconds(passage.endSampleExclusive);+ return {+ text: words.map((word) => word.text).join(" "),+ ...(start === undefined ? {} : { start_seconds: start }),+ ...(end === undefined ? {} : { end_seconds: end }),+ words,+ };+ });+}++async function transcribe(argv: readonly string[], io: CliIo, environment: CreationEnvironment): Promise<void> {+ const parsed = parseArguments(argv, ["--language", "--to", "--runtime"]);+ assert(parsed.positionals.length === 1, "transcribe takes exactly one audio or video file");+ const source = resolve(environment.cwd, parsed.positionals[0]!);+ const language = parsed.options.get("--language") ?? "en";+ assert(language === "en" || language === "zh" || language === "es", "--language must be en, zh or es");+ const to = await destination(parsed, environment.cwd);+ const { profile, host } = await environment.openHost(parsed.options.get("--runtime"));+ const provider = await selectedProvider(host, whisperXCapabilities.alignment, profile);+ if (!parsed.json) io.write(`Transcribing through ${providerLine(provider)}\n`);+ const evidence = await speechEvidenceBytes(source);+ const resources = new MemoryResourceStore();+ const artifact = await resources.put(evidence.bytes, "audio/wav");+ const audio = sealSpeechEvidenceAudio({ artifact, sampleFrames: evidence.sampleFrames });+ const need: Need = {+ id: "need:hypit-transcribe",+ capability: whisperXCapabilities.alignment,+ returns: speechEvidenceTypes.alignedTranscript,+ constraints: whisperXRequestForEvidenceAudio(audio, { language: language as WhisperXLanguage }),+ result: "record:hypit-transcribe",+ };+ const fulfillment = await host.invoke(need, resources);+ assert(fulfillment.value.kind === "inline", "the transcript came back by reference");+ const passages = passagesInSeconds(fulfillment.value.value as unknown as AlignedTranscriptEvidence);+ const words = passages.reduce((total, passage) => total + passage.words.length, 0);+ const file = {+ format: "hypit.transcript@1",+ source,+ language,+ audio_seconds: round(evidence.sampleFrames / EVIDENCE_SAMPLE_RATE),+ passages,+ };+ await writeNew(to, `${JSON.stringify(file, null, 2)}\n`);+ const view = {+ format: "hypit.video-cli-transcribe@1",+ source,+ language,+ audio_seconds: file.audio_seconds,+ extracted: evidence.extracted,+ ...providerView(provider),+ passages: passages.length,+ words,+ path: to,+ };+ if (parsed.json) io.write(`${JSON.stringify(view, null, 2)}\n`);+ else io.write(`✓ Transcript written\n\n ${words} words in ${passages.length} passage${passages.length === 1 ? "" : "s"} over ${file.audio_seconds}s\n ${to}\n`);+}++// ---------------------------------------------------------------------------------------------------+// speak++async function speak(argv: readonly string[], io: CliIo, environment: CreationEnvironment): Promise<void> {+ const parsed = parseArguments(argv, ["--text", "--voice", "--to", "--runtime"]);+ assert(parsed.positionals.length === 0, `speak takes no positional arguments; received ${parsed.positionals.join(" ")}`);+ const text = await textOrFile(required(parsed, "--text", "the words to speak, or a file holding them"), "--text", environment.cwd);+ const voice = await textOrFile(required(parsed, "--voice", "a description of the voice, or a file holding it"), "--voice", environment.cwd);+ const to = await destination(parsed, environment.cwd);+ const endpoint = mimoTtsEndpoints.voiceDesign;+ assert(endpoint !== undefined, "@hypit/mimo-tts declares no voice-design endpoint");+ const { profile, host } = await environment.openHost(parsed.options.get("--runtime"));+ const provider = await selectedProvider(host, endpoint.capability, profile);+ if (!parsed.json) io.write(`Speaking with ${endpoint.ports.model} through ${providerLine(provider)}\n`);+ const resources = new MemoryResourceStore();+ const need: Need = {+ id: "need:hypit-speak",+ capability: endpoint.capability,+ returns: endpoint.returns,+ constraints: sealMimoTtsRequest("mimo-v2.5-tts-voicedesign", { text: [text], voiceDescription: [voice] }) as unknown as CanonicalValue,+ result: "record:hypit-speak",+ };+ const fulfillment = await host.invoke(need, resources);+ assert(fulfillment.value.kind === "inline", "the speech came back by reference");+ verifyGeneratedAudioSet(fulfillment.value.value);+ const [audio] = (fulfillment.value.value as unknown as GeneratedAudioSet).audios;+ assert(audio !== undefined, "the model returned no audio");+ const bytes = await resources.get(audio.resource);+ assert(bytes !== undefined, `generated audio ${audio.resource} was not stored`);+ await writeNew(to, bytes);+ const durationSeconds = await audioSeconds(to, bytes);+ const view = {+ format: "hypit.video-cli-speak@1",+ model: endpoint.ports.model,+ ...providerView(provider),+ mediaType: audio.mediaType,+ bytes: bytes.byteLength,+ ...(durationSeconds === undefined ? {} : { duration_seconds: durationSeconds }),+ path: to,+ };+ if (parsed.json) io.write(`${JSON.stringify(view, null, 2)}\n`);+ else {+ io.write(`✓ Speech written\n\n ${audio.mediaType} · ${bytes.byteLength} bytes${durationSeconds === undefined ? "" : ` · ${durationSeconds}s`}\n ${to}\n`);+ if (durationSeconds === undefined) io.write(" (duration unknown: ffprobe is unavailable; measure the file before writing a literal)\n");+ }+}++// ---------------------------------------------------------------------------------------------------+// Entry++export function writeCreationHelp(io: CliIo, topic?: CreationCommand): void {+ const sections: Record<CreationCommand, readonly string[]> = {+ observe: [+ "hypit observe",+ "Look at pictures, clips or recordings with the Gemini Endpoint of the selected Runtime Profile.",+ "",+ " hypit observe <media…> --instruction <text|file> --prompt <text|file> --to <file>",+ " [--model <gemini model>] [--runtime <profile>]",+ "",+ "Writes the observation as text to --to. One immediate request; no Build, Result or state.",+ ],+ transcribe: [+ "hypit transcribe",+ "Hear a recording with the whisperx-alignment Endpoint of the selected Runtime Profile.",+ "",+ " hypit transcribe <audio|video> --to <transcript.json> [--language en|zh|es] [--runtime <profile>]",+ "",+ "Extracts 16 kHz mono speech audio with ffmpeg and writes every word with its start and end in",+ "seconds. One immediate request; no Build, Result or state.",+ ],+ speak: [+ "hypit speak",+ "Speak a line with the MiMo VoiceDesign Endpoint of the selected Runtime Profile.",+ "",+ " hypit speak --text <text|file> --voice <description|file> --to <audio file> [--runtime <profile>]",+ "",+ "Writes the audio and reports its duration, so the author can write that duration as a literal.",+ "For a voice-over video this is the A-roll: speak first, measure, then author B-roll that covers",+ "every passage. Where a real presenter is the A-roll, do not replace them with this.",+ ],+ };+ const chosen = topic === undefined ? creationCommands : [topic];+ io.write(`${chosen.map((item) => sections[item].join("\n")).join("\n\n")}\n\n`+ + "Each command names the Endpoint and its price page before it runs; the Profile comes from --runtime\n"+ + "or the project's `hypit runtime use` selection.\n");+}++export async function runCreationCli(argv: readonly string[], io: CliIo, environment: CreationEnvironment = creationEnvironment()): Promise<void> {+ const command = argv[0];+ assert(isCreationCommand(command), `unknown creation command ${command}`);+ if (argv.includes("--help")) { writeCreationHelp(io, command); return; }+ if (command === "observe") await observe(argv, io, environment);+ else if (command === "transcribe") await transcribe(argv, io, environment);+ else await speak(argv, io, environment);+}diff --git a/packages/video-cli/src/index.ts b/packages/video-cli/src/index.tsindex 7880f312..7a55d30d 100644--- a/packages/video-cli/src/index.ts+++ b/packages/video-cli/src/index.ts@@ -7,6 +7,7 @@ import {import { hypitHostPackageRoot } from "@hypit/runtime-host-node";import type { LoadedPackage } from "@hypit/package-loader-node";+import { isCreationCommand, runCreationCli } from "./creation.js";import { videoCliDistribution } from "./distribution.js";export {@@ -14,6 +15,8 @@ export {} from "./compiler.js";export { videoCliDistribution } from "./distribution.js";export { discoverVideoSourcePackages } from "./package-selection.js";+export { creationCommands, isCreationCommand, runCreationCli, writeCreationHelp } from "./creation.js";+export type { CreationCommand, CreationEnvironment, CreationHost } from "./creation.js";/** The project's selected Runtime Profile, read the way `hypit` reads it, for tools that run beside the CLI. */export { findRuntimeProfile } from "@hypit/cli";@@ -26,6 +29,7 @@ export function runVideoCli(? []: [videoCliDistribution.packageRoot]);installExternalPackageResolution([hypitHostPackageRoot()]);+ if (isCreationCommand(argv[0])) return runCreationCli(argv, io);return runCli(argv, io, {...videoCliDistribution,bootstrapPackages: packages,diff --git a/packages/video-cli/test/creation.test.ts b/packages/video-cli/test/creation.test.tsnew file mode 100644index 00000000..bb376427--- /dev/null+++ b/packages/video-cli/test/creation.test.ts@@ -0,0 +1,161 @@+import assert from "node:assert/strict";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";++import { canonicalize } from "@hypit/protocol";+import type { CapabilityRef, Need } from "@hypit/protocol";+import type { ResourceStore } from "@hypit/runtime";+import { sealAlignedTranscriptEvidence } from "@hypit/speech-evidence";+import { sealText } from "@hypit/text";+import { interpretWhisperXTranscript } from "@hypit/whisperx";+import type { WhisperXAlignmentRequest } from "@hypit/whisperx";++import { runCreationCli } from "../src/creation.js";+import type { CreationEnvironment, CreationHost } from "../src/creation.js";++/** 16 kHz mono PCM s16 WAV: the canonical evidence shape, so no ffmpeg runs in these tests. */+function wav(sampleFrames: number): Uint8Array {+ const bytes = new Uint8Array(44 + sampleFrames * 2);+ const view = new DataView(bytes.buffer);+ const write = (offset: number, value: string): void => {+ for (let index = 0; index < value.length; index += 1) bytes[offset + index] = value.charCodeAt(index);+ };+ write(0, "RIFF"); view.setUint32(4, bytes.byteLength - 8, true); write(8, "WAVE");+ write(12, "fmt "); view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true);+ view.setUint32(24, 16_000, true); view.setUint32(28, 32_000, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true);+ write(36, "data"); view.setUint32(40, sampleFrames * 2, true);+ return bytes;+}++const page = { kind: "page" as const, url: "https://prices.example/models" };++/** A host whose Profile serves everything through one paid Endpoint, answering each Need in kind. */+function host(seen: Need[]): CreationHost {+ return {+ providers: async (capabilities: readonly CapabilityRef[]) => capabilities.map((capability) => ({+ capability, status: "resolved" as const, endpoint: "paid.default", use: "@hypit/provider-example", pricing: page,+ })),+ invoke: async (need: Need, resources: ResourceStore) => {+ seen.push(need);+ if (need.capability.name === "whisperx-alignment") {+ const request = need.constraints as unknown as WhisperXAlignmentRequest;+ return { value: { kind: "inline" as const, value: canonicalize(sealAlignedTranscriptEvidence({+ passages: interpretWhisperXTranscript({+ segments: [{ start: 0, end: 1.5, words: [{ text: "hello", start: 0.1, end: 0.5 }, { text: "world", start: 0.9, end: 1.4 }] }],+ }, request.sampleFrames),+ })) } };+ }+ if (need.capability.name === "mimo-v2.5-tts-voicedesign") {+ const artifact = await resources.put(wav(24_000), "audio/wav");+ return { value: { kind: "inline" as const, value: canonicalize({ audios: [artifact] }) } };+ }+ const request = need.constraints as { readonly prompt: string; readonly media: readonly unknown[] };+ return { value: { kind: "inline" as const, value: canonicalize(sealText(`saw ${request.media.length} media; asked: ${request.prompt}`)) } };+ },+ };+}++function environment(cwd: string, seen: Need[]): CreationEnvironment {+ return { cwd, openHost: async () => ({ profile: join(cwd, "hypit.runtime.json"), host: host(seen) }) };+}++function capture() {+ let output = "";+ return { io: { write: (text: string) => { output += text; } }, text: () => output };+}++test("observe seals one Gemini request from the media and writes the answer beside the Provider it named", async () => {+ const root = await mkdtemp(join(tmpdir(), "hypit-observe-"));+ const seen: Need[] = [];+ try {+ await writeFile(join(root, "frame.png"), Uint8Array.from([137, 80, 78, 71]));+ await writeFile(join(root, "prompt.txt"), "What is the presenter holding?\n", "utf8");+ const out = capture();+ await runCreationCli([+ "observe", "frame.png", "--instruction", "You only observe.", "--prompt", "prompt.txt", "--to", "notes/frame.md", "--json",+ ], out.io, environment(root, seen));+ const view = JSON.parse(out.text()) as Record<string, unknown>;+ assert.equal(view.endpoint, "paid.default");+ assert.deepEqual(view.pricing, page);+ assert.equal(await readFile(join(root, "notes", "frame.md"), "utf8"), "saw 1 media; asked: What is the presenter holding?\n");+ assert.equal(seen[0]?.capability.name, "gemini-3.1-pro");+ const constraints = seen[0]!.constraints as { readonly instruction: string; readonly media: readonly { readonly artifact: { readonly mediaType: string } }[] };+ assert.equal(constraints.instruction, "You only observe.");+ assert.equal(constraints.media[0]?.artifact.mediaType, "image/png");++ const again = capture();+ await assert.rejects(runCreationCli([+ "observe", "frame.png", "--instruction", "x", "--prompt", "y", "--to", "notes/frame.md",+ ], again.io, environment(root, seen)), /already exists/u);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test("transcribe writes every word in seconds from the Profile's alignment Endpoint", async () => {+ const root = await mkdtemp(join(tmpdir(), "hypit-transcribe-"));+ const seen: Need[] = [];+ try {+ await writeFile(join(root, "speech.wav"), wav(32_000));+ const out = capture();+ await runCreationCli(["transcribe", "speech.wav", "--to", "speech.json", "--language", "en"], out.io, environment(root, seen));+ assert.match(out.text(), /paid\.default \(@hypit\/provider-example\) · https:\/\/prices\.example\/models/u);+ assert.match(out.text(), /2 words in 1 passage over 2s/u);+ const file = JSON.parse(await readFile(join(root, "speech.json"), "utf8")) as {+ readonly audio_seconds: number; readonly passages: readonly { readonly words: readonly { readonly text: string; readonly start_seconds: number }[] }[];+ };+ assert.equal(file.audio_seconds, 2);+ assert.deepEqual(file.passages[0]?.words.map((word) => [word.text, word.start_seconds]), [["hello", 0.1], ["world", 0.9]]);+ const request = seen[0]!.constraints as unknown as WhisperXAlignmentRequest;+ assert.equal(request.language, "en");+ assert.equal(request.sampleFrames, 32_000);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test("speak writes the audio and its duration so the author can write a literal", async () => {+ const root = await mkdtemp(join(tmpdir(), "hypit-speak-"));+ const seen: Need[] = [];+ try {+ await writeFile(join(root, "line.txt"), "Welcome back to the channel.\n", "utf8");+ const out = capture();+ await runCreationCli([+ "speak", "--text", "line.txt", "--voice", "warm, unhurried, mid-thirties", "--to", "voice/intro.wav", "--json",+ ], out.io, environment(root, seen));+ const view = JSON.parse(out.text()) as Record<string, unknown>;+ assert.equal(view.model, "mimo-v2.5-tts-voicedesign");+ assert.equal(view.mediaType, "audio/wav");+ assert.equal(view.duration_seconds, 1.5);+ assert.equal((await readFile(join(root, "voice", "intro.wav"))).byteLength, 44 + 24_000 * 2);+ const ports = seen[0]!.constraints as { readonly ports?: Record<string, unknown> };+ assert.equal(JSON.stringify(ports).includes("Welcome back to the channel."), true);+ assert.equal(JSON.stringify(ports).includes("warm, unhurried"), true);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test("a Profile that does not serve the capability stops before anything is spent", async () => {+ const root = await mkdtemp(join(tmpdir(), "hypit-creation-unserved-"));+ try {+ await writeFile(join(root, "speech.wav"), wav(16_000));+ let invoked = false;+ const unserved: CreationEnvironment = {+ cwd: root,+ openHost: async () => ({ profile: join(root, "hypit.runtime.json"), host: {+ providers: async (capabilities) => capabilities.map((capability) => ({ capability, status: "unresolved" as const })),+ invoke: async () => { invoked = true; throw new Error("must not be reached"); },+ } }),+ };+ await assert.rejects(+ runCreationCli(["transcribe", "speech.wav", "--to", "speech.json"], capture().io, unserved),+ /No Endpoint in .*hypit\.runtime\.json serves @hypit\/whisperx@1#whisperx-alignment; hypit plan --runtime/u,+ );+ assert.equal(invoked, false);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 9d2937ae..33794a13 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -3394,21 +3394,51 @@ importers:'@hypit/credential-store-os':specifier: workspace:*version: link:../credential-store-os+ '@hypit/driver-node':+ specifier: workspace:*+ version: link:../driver-node+ '@hypit/gemini':+ specifier: workspace:*+ version: link:../gemini+ '@hypit/generation':+ specifier: workspace:*+ version: link:../generation'@hypit/markup':specifier: workspace:*version: link:../markup+ '@hypit/mimo-tts':+ specifier: workspace:*+ version: link:../mimo-tts'@hypit/package-loader-node':specifier: workspace:*version: link:../package-loader-node+ '@hypit/protocol':+ specifier: workspace:*+ version: link:../protocol'@hypit/provider-hypihub':specifier: workspace:*version: link:../provider-hypihub+ '@hypit/runtime':+ specifier: workspace:*+ version: link:../runtime'@hypit/runtime-host-node':specifier: workspace:*version: link:../runtime-host-node'@hypit/runtime-local':specifier: workspace:*version: link:../runtime-local+ '@hypit/speech':+ specifier: workspace:*+ version: link:../speech+ '@hypit/speech-evidence':+ specifier: workspace:*+ version: link:../speech-evidence+ '@hypit/text':+ specifier: workspace:*+ version: link:../text+ '@hypit/whisperx':+ specifier: workspace:*+ version: link:../whisperx'@hypit/workspace-fs-node':specifier: workspace:*version: link:../workspace-fs-node