Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(cli): the api provider speaks the HeyGen audio analysis contract (#5641)
Xuanru Li124492e427d1
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex 5078c9413c..fb4aea133b 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -566,7 +566,10 @@ answers:| describe | `api`, `apple-sound` |- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and- `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,+ `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono+ WAV. Channels are averaged unless cancellation removes over 90% of the loudest+ channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each+ request; the answer may come wrapped in `{ "data": … }`. Every beat,downbeat or section boundary it answers moves onto the nearest beat signalprocessing measured, when one is within a quarter beat; the rest are listed in`snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.tsindex 8488f0bb76..2fb5d919c2 100644--- a/packages/cli/src/audio/analysis/providers/api.test.ts+++ b/packages/cli/src/audio/analysis/providers/api.test.ts@@ -11,6 +11,7 @@ import { analyzeSignal } from "../analyze.js";import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js";const RATE = 22_050;+const UPLOAD_RATE = 16_000;/** Clicks every half second from 0.25 s: a grid the signal tracker finds. */function clicks(seconds: number): AudioSignal {@@ -27,7 +28,9 @@ function clicks(seconds: number): AudioSignal {interface Received {method: string;+ apiKey: string | undefined;authorization: string | undefined;+ idempotencyKey: string | undefined;request: AudioApiRequest;audio: { type: string; bytes: Uint8Array };}@@ -38,6 +41,7 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = ()status: 500,body: {},});+let enveloped = true;async function readForm(req: IncomingMessage) {const chunks: Buffer[] = [];@@ -57,9 +61,16 @@ async function readForm(req: IncomingMessage) {beforeAll(async () => {server = createServer(async (req, res) => {const form = await readForm(req);- received.push({ method: req.method!, authorization: req.headers.authorization, ...form });+ received.push({+ method: req.method!,+ apiKey: req.headers["x-api-key"] as string | undefined,+ authorization: req.headers.authorization,+ idempotencyKey: req.headers["idempotency-key"] as string | undefined,+ ...form,+ });const { status, body } = reply(form.request);- res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));+ const sent = status === 200 && enveloped ? { data: body } : body;+ res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(sent));});await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));});@@ -68,6 +79,7 @@ afterAll(() => new Promise<void>((done) => server.close(() => done())));afterEach(() => {received = [];+ enveloped = true;delete process.env.HYPERFRAMES_AUDIO_API_URL;delete process.env.HYPERFRAMES_AUDIO_API_KEY;});@@ -92,6 +104,34 @@ describe("apiProvider", () => {expect(apiProvider.available!()).toEqual({ ok: true });});+ it("refuses redirects without sending the API key to another origin", async () => {+ const forwarded: string[] = [];+ const destination = createServer((req, res) => {+ forwarded.push(req.headers["x-api-key"]?.toString() ?? "");+ res.writeHead(200).end("{}");+ });+ await new Promise<void>((done) => destination.listen(0, "127.0.0.1", done));+ const address = destination.address();+ if (!address || typeof address === "string") throw new Error("No destination port");+ const redirect = createServer((_req, res) => {+ res.writeHead(307, { location: `http://127.0.0.1:${address.port}/` }).end();+ });+ await new Promise<void>((done) => redirect.listen(0, "127.0.0.1", done));+ try {+ const source = redirect.address();+ if (!source || typeof source === "string") throw new Error("No redirect port");+ process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${source.port}/`;+ process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key";+ await expect(apiProvider.analyze("beats", clicks(1), {})).rejects.toThrow();+ expect(forwarded).toEqual([]);+ } finally {+ await Promise.all([+ new Promise<void>((done) => redirect.close(() => done())),+ new Promise<void>((done) => destination.close(() => done())),+ ]);+ }+ });+it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => {configure();const signal = clicks(12);@@ -103,6 +143,7 @@ describe("apiProvider", () => {body: {version: 1,source: "stand-in",+ time_resolution_seconds: 1 / 32,beats: { times: answered },downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 },} satisfies AudioApiResponse,@@ -114,15 +155,18 @@ describe("apiProvider", () => {expect(received).toHaveLength(1);const [call] = received;- expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" });+ expect(call).toMatchObject({ method: "POST", apiKey: "test-key", authorization: undefined });+ expect(call!.idempotencyKey).toMatch(+ /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/,+ );expect(call!.request).toEqual({version: 1,capabilities: ["beats", "downbeats"],- audio: { sampleRate: RATE, channels: 1, duration: 12 },+ audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 },});expect(call!.audio.type).toBe("audio/wav");expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");- expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2);+ expect(call!.audio.bytes.length).toBe(44 + 12 * UPLOAD_RATE * 2);expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" });const beats = analysis.values.beats as { times: number[]; snap: object; source: string };@@ -201,6 +245,7 @@ describe("apiProvider", () => {signalProvider,]);expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);+ expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2);expect(analysis.values.sections).toMatchObject({spans: [{ start: 0, end: beat, label: "intro" },@@ -235,10 +280,96 @@ describe("apiProvider", () => {expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape");});- it("never sends the key anywhere but the authorization header", async () => {+ it("uploads the channels averaged to one at 16 kHz, with nothing above 8 kHz folded back in", async () => {+ configure();+ const rate = 44_100;+ const tone = (hz: number) =>+ Float32Array.from(+ { length: 2 * rate },+ (_, k) => 0.3 * Math.sin((2 * Math.PI * hz * k) / rate),+ );+ const [low, side, high] = [tone(440), tone(2_000), tone(9_000)];+ const left = low.map((v, k) => v + side[k]! + high[k]!);+ const right = low.map((v, k) => v - side[k]! + high[k]!);+ reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });+ await analyzeSignal(+ { sampleRate: rate, channels: [left, right] },+ { capabilities: ["describe"] },+ [apiProvider],+ );++ const wav = Buffer.from(received[0]!.audio.bytes);+ expect([wav.readUInt16LE(22), wav.readUInt32LE(24), wav.readUInt16LE(34)]).toEqual([+ 1,+ UPLOAD_RATE,+ 16,+ ]);+ const samples = Array.from(+ { length: (wav.length - 44) / 2 },+ (_, k) => wav.readInt16LE(44 + 2 * k) / 32767,+ );+ const level = (hz: number) => {+ let re = 0;+ let im = 0;+ samples.forEach((v, k) => {+ re += v * Math.cos((2 * Math.PI * hz * k) / UPLOAD_RATE);+ im += v * Math.sin((2 * Math.PI * hz * k) / UPLOAD_RATE);+ });+ return (2 * Math.hypot(re, im)) / samples.length;+ };+ expect(level(440)).toBeCloseTo(0.3, 2);+ expect(level(2_000)).toBeLessThan(0.005);+ expect(level(7_000)).toBeLessThan(0.005);+ });++ it.each([+ { phase: Math.PI, quiet: 1, expected: 0.7 },+ { phase: Math.PI * 0.9, quiet: 1, expected: 0.7 },+ { phase: Math.PI, quiet: 0.8, expected: 0.7 },+ { phase: Math.PI / 2, quiet: 1, expected: 0.7 / Math.sqrt(2) },+ { phase: 0, quiet: 1, expected: 0.7 },+ ])(+ "preserves stereo energy for phase $phase and channel gain $quiet",+ async ({ phase, quiet, expected }) => {+ configure();+ const rate = 44_100;+ const left = Float32Array.from(+ { length: rate },+ (_, k) => 0.7 * quiet * Math.sin((2 * Math.PI * 440 * k) / rate),+ );+ const right = Float32Array.from(+ { length: rate },+ (_, k) => 0.7 * Math.sin((2 * Math.PI * 440 * k) / rate + phase),+ );+ reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });+ await analyzeSignal(+ { sampleRate: rate, channels: [left, right] },+ { capabilities: ["describe"] },+ [apiProvider],+ );+ const wav = Buffer.from(received[0]!.audio.bytes);+ expect(wav.readUInt32LE(24)).toBe(UPLOAD_RATE);+ let energy = 0;+ for (let k = 0; k < UPLOAD_RATE; k++) energy += (wav.readInt16LE(44 + 2 * k) / 32767) ** 2;+ expect(Math.sqrt((2 * energy) / UPLOAD_RATE)).toBeCloseTo(expected, 2);+ },+ );++ it("reads an answer sent without the data envelope too", async () => {+ configure();+ enveloped = false;+ reply = () => ({ status: 200, body: { version: 1, describe: { text: "bare", tags: [] } } });+ const analysis = await analyzeSignal(clicks(2), { capabilities: ["describe"] }, [apiProvider]);+ expect(analysis.values.describe).toEqual({ text: "bare", tags: [] });+ });++ it("never sends the key anywhere but the x-api-key header", async () => {configure();reply = () => ({ status: 401, body: { error: { message: "bad key" } } });const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]);expect(JSON.stringify(analysis)).not.toContain("test-key");+ const [call] = received;+ expect(call).toMatchObject({ apiKey: "test-key", authorization: undefined });+ expect(JSON.stringify(call!.request)).not.toContain("test-key");});});diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.tsindex 5659f19206..8db80a854c 100644--- a/packages/cli/src/audio/analysis/providers/api.ts+++ b/packages/cli/src/audio/analysis/providers/api.ts@@ -1,3 +1,4 @@+import { randomUUID } from "node:crypto";import {medianInterval,rhythm,@@ -12,14 +13,14 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared./*** The audio analysis service contract.*- * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with- * `Authorization: Bearer <HYPERFRAMES_AUDIO_API_KEY>` and a `multipart/form-data` body of two parts:+ * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with `x-api-key: <HYPERFRAMES_AUDIO_API_KEY>`,+ * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts:* - `request`: `application/json`, an {@link AudioApiRequest};- * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded.+ * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz.*- * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the- * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure;- * its body may be `{ "error": { "message": string } }`.+ * Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read+ * too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left+ * out of the response. Any other status is a failure; its body may be `{ "error": { "message": string } }`.*/export interface AudioApiRequest {version: 1;@@ -33,6 +34,8 @@ export interface AudioApiResponse {version: 1;/** What answered, as the service wants it reported. */source?: string;+ /** The finest step the service's times resolve, in seconds. */+ time_resolution_seconds?: number;beats?: { times: number[] };downbeats?: { times: number[]; meter?: number | null };sections?: { spans: { start: number; end: number; label?: string }[] };@@ -43,6 +46,68 @@ const URL_ENV = "HYPERFRAMES_AUDIO_API_URL";const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";const DEFAULT_TIMEOUT_MS = 120_000;+const UPLOAD_RATE = 16_000;+const SINC_ZEROS = 16;+const ROLLOFF = 0.9;+const SINC_STEPS = 512;+const HANN_SINC = Float32Array.from({ length: SINC_ZEROS * SINC_STEPS + 2 }, (_, k) => {+ const x = k / SINC_STEPS;+ if (x >= SINC_ZEROS) return 0;+ const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);+ return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / SINC_ZEROS));+});++/** Windowed-sinc resampling with its cutoff just under the lower Nyquist, so nothing above it folds back in. */+function resample(input: Float32Array, from: number, to: number): Float32Array {+ if (from === to) return input;+ const step = from / to;+ const scale = Math.min(1, to / from) * ROLLOFF;+ const reach = SINC_ZEROS / scale;+ const output = new Float32Array(Math.round(input.length / step));+ for (let i = 0; i < output.length; i++) {+ const center = i * step;+ const last = Math.min(input.length - 1, Math.floor(center + reach));+ let sum = 0;+ let weights = 0;+ for (let n = Math.max(0, Math.ceil(center - reach)); n <= last; n++) {+ const at = Math.abs(n - center) * scale * SINC_STEPS;+ const k = Math.floor(at);+ const weight = HANN_SINC[k]! + (at - k) * (HANN_SINC[k + 1]! - HANN_SINC[k]!);+ sum += input[n]! * weight;+ weights += weight;+ }+ output[i] = weights ? sum / weights : 0;+ }+ return output;+}++/** Preserve a channel when destructive interference removes over 90% of its energy. */+function monoInput(channels: readonly Float32Array[]): Float32Array {+ const mono = new Float32Array(channels[0]?.length ?? 0);+ let loudest: Float32Array = mono;+ let loudestEnergy = 0;+ for (const channel of channels) {+ let energy = 0;+ for (let i = 0; i < mono.length; i++) {+ const value = channel[i]!;+ mono[i]! += value / channels.length;+ energy += value * value;+ }+ if (energy > loudestEnergy) {+ loudest = channel;+ loudestEnergy = energy;+ }+ }+ const monoEnergy = mono.reduce((energy, value) => energy + value * value, 0);+ return monoEnergy < loudestEnergy * 0.1 ? loudest : mono;+}++function uploaded(signal: AudioSignal): AudioSignal {+ return {+ sampleRate: UPLOAD_RATE,+ channels: [resample(monoInput(signal.channels), signal.sampleRate, UPLOAD_RATE)],+ };+}/** Beats and downbeats come from one request; the others each from their own. */const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] =>@@ -90,16 +155,17 @@ async function failure(response: Response): Promise<Error> {return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);}-async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {+async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) {const { url, key, timeoutMs } = config();if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);+ const signal = uploaded(analyzed);const request: AudioApiRequest = {version: 1,capabilities,audio: {sampleRate: signal.sampleRate,- channels: signal.channels.length,- duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate,+ channels: 1,+ duration: signal.channels[0]!.length / signal.sampleRate,},};const form = new FormData();@@ -111,12 +177,14 @@ async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {);const response = await fetch(url, {method: "POST",- headers: { authorization: `Bearer ${key}` },+ redirect: "error",+ headers: { "x-api-key": key, "idempotency-key": randomUUID() },body: form,signal: AbortSignal.timeout(timeoutMs),});if (!response.ok) throw await failure(response);- return (await response.json()) as AudioApiResponse;+ const body = (await response.json()) as { data?: AudioApiResponse } | null;+ return (body?.data ?? body ?? {}) as AudioApiResponse;}const requests = new Map<string, (signal: AudioSignal) => Promise<AudioApiResponse>>();