Skip to content

Write HTML. Render video. Built for agents.

feat(cli): the api provider speaks the HeyGen audio analysis contract (#5641)

Xuanru Li124492e427d1
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index 5078c9413c..fb4aea133b 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -566,7 +566,10 @@ answers:
| describe | `api`, `apple-sound` |
- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and
- `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,
+ `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono
+ WAV. Channels are averaged unless cancellation removes over 90% of the loudest
+ channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each
+ request; the answer may come wrapped in `{ "data": … }`. Every beat,
downbeat or section boundary it answers moves onto the nearest beat signal
processing measured, when one is within a quarter beat; the rest are listed in
`snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.
diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts
index 8488f0bb76..2fb5d919c2 100644
--- a/packages/cli/src/audio/analysis/providers/api.test.ts
+++ b/packages/cli/src/audio/analysis/providers/api.test.ts
@@ -11,6 +11,7 @@ import { analyzeSignal } from "../analyze.js";
import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js";
const RATE = 22_050;
+const UPLOAD_RATE = 16_000;
/** Clicks every half second from 0.25 s: a grid the signal tracker finds. */
function clicks(seconds: number): AudioSignal {
@@ -27,7 +28,9 @@ function clicks(seconds: number): AudioSignal {
interface Received {
method: string;
+ apiKey: string | undefined;
authorization: string | undefined;
+ idempotencyKey: string | undefined;
request: AudioApiRequest;
audio: { type: string; bytes: Uint8Array };
}
@@ -38,6 +41,7 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = ()
status: 500,
body: {},
});
+let enveloped = true;
async function readForm(req: IncomingMessage) {
const chunks: Buffer[] = [];
@@ -57,9 +61,16 @@ async function readForm(req: IncomingMessage) {
beforeAll(async () => {
server = createServer(async (req, res) => {
const form = await readForm(req);
- received.push({ method: req.method!, authorization: req.headers.authorization, ...form });
+ received.push({
+ method: req.method!,
+ apiKey: req.headers["x-api-key"] as string | undefined,
+ authorization: req.headers.authorization,
+ idempotencyKey: req.headers["idempotency-key"] as string | undefined,
+ ...form,
+ });
const { status, body } = reply(form.request);
- res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));
+ const sent = status === 200 && enveloped ? { data: body } : body;
+ res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(sent));
});
await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));
});
@@ -68,6 +79,7 @@ afterAll(() => new Promise<void>((done) => server.close(() => done())));
afterEach(() => {
received = [];
+ enveloped = true;
delete process.env.HYPERFRAMES_AUDIO_API_URL;
delete process.env.HYPERFRAMES_AUDIO_API_KEY;
});
@@ -92,6 +104,34 @@ describe("apiProvider", () => {
expect(apiProvider.available!()).toEqual({ ok: true });
});
+ it("refuses redirects without sending the API key to another origin", async () => {
+ const forwarded: string[] = [];
+ const destination = createServer((req, res) => {
+ forwarded.push(req.headers["x-api-key"]?.toString() ?? "");
+ res.writeHead(200).end("{}");
+ });
+ await new Promise<void>((done) => destination.listen(0, "127.0.0.1", done));
+ const address = destination.address();
+ if (!address || typeof address === "string") throw new Error("No destination port");
+ const redirect = createServer((_req, res) => {
+ res.writeHead(307, { location: `http://127.0.0.1:${address.port}/` }).end();
+ });
+ await new Promise<void>((done) => redirect.listen(0, "127.0.0.1", done));
+ try {
+ const source = redirect.address();
+ if (!source || typeof source === "string") throw new Error("No redirect port");
+ process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${source.port}/`;
+ process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key";
+ await expect(apiProvider.analyze("beats", clicks(1), {})).rejects.toThrow();
+ expect(forwarded).toEqual([]);
+ } finally {
+ await Promise.all([
+ new Promise<void>((done) => redirect.close(() => done())),
+ new Promise<void>((done) => destination.close(() => done())),
+ ]);
+ }
+ });
+
it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => {
configure();
const signal = clicks(12);
@@ -103,6 +143,7 @@ describe("apiProvider", () => {
body: {
version: 1,
source: "stand-in",
+ time_resolution_seconds: 1 / 32,
beats: { times: answered },
downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 },
} satisfies AudioApiResponse,
@@ -114,15 +155,18 @@ describe("apiProvider", () => {
expect(received).toHaveLength(1);
const [call] = received;
- expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" });
+ expect(call).toMatchObject({ method: "POST", apiKey: "test-key", authorization: undefined });
+ expect(call!.idempotencyKey).toMatch(
+ /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/,
+ );
expect(call!.request).toEqual({
version: 1,
capabilities: ["beats", "downbeats"],
- audio: { sampleRate: RATE, channels: 1, duration: 12 },
+ audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 },
});
expect(call!.audio.type).toBe("audio/wav");
expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");
- expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2);
+ expect(call!.audio.bytes.length).toBe(44 + 12 * UPLOAD_RATE * 2);
expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" });
const beats = analysis.values.beats as { times: number[]; snap: object; source: string };
@@ -201,6 +245,7 @@ describe("apiProvider", () => {
signalProvider,
]);
expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);
+ expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2);
expect(analysis.values.sections).toMatchObject({
spans: [
{ start: 0, end: beat, label: "intro" },
@@ -235,10 +280,96 @@ describe("apiProvider", () => {
expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape");
});
- it("never sends the key anywhere but the authorization header", async () => {
+ it("uploads the channels averaged to one at 16 kHz, with nothing above 8 kHz folded back in", async () => {
+ configure();
+ const rate = 44_100;
+ const tone = (hz: number) =>
+ Float32Array.from(
+ { length: 2 * rate },
+ (_, k) => 0.3 * Math.sin((2 * Math.PI * hz * k) / rate),
+ );
+ const [low, side, high] = [tone(440), tone(2_000), tone(9_000)];
+ const left = low.map((v, k) => v + side[k]! + high[k]!);
+ const right = low.map((v, k) => v - side[k]! + high[k]!);
+ reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });
+ await analyzeSignal(
+ { sampleRate: rate, channels: [left, right] },
+ { capabilities: ["describe"] },
+ [apiProvider],
+ );
+
+ const wav = Buffer.from(received[0]!.audio.bytes);
+ expect([wav.readUInt16LE(22), wav.readUInt32LE(24), wav.readUInt16LE(34)]).toEqual([
+ 1,
+ UPLOAD_RATE,
+ 16,
+ ]);
+ const samples = Array.from(
+ { length: (wav.length - 44) / 2 },
+ (_, k) => wav.readInt16LE(44 + 2 * k) / 32767,
+ );
+ const level = (hz: number) => {
+ let re = 0;
+ let im = 0;
+ samples.forEach((v, k) => {
+ re += v * Math.cos((2 * Math.PI * hz * k) / UPLOAD_RATE);
+ im += v * Math.sin((2 * Math.PI * hz * k) / UPLOAD_RATE);
+ });
+ return (2 * Math.hypot(re, im)) / samples.length;
+ };
+ expect(level(440)).toBeCloseTo(0.3, 2);
+ expect(level(2_000)).toBeLessThan(0.005);
+ expect(level(7_000)).toBeLessThan(0.005);
+ });
+
+ it.each([
+ { phase: Math.PI, quiet: 1, expected: 0.7 },
+ { phase: Math.PI * 0.9, quiet: 1, expected: 0.7 },
+ { phase: Math.PI, quiet: 0.8, expected: 0.7 },
+ { phase: Math.PI / 2, quiet: 1, expected: 0.7 / Math.sqrt(2) },
+ { phase: 0, quiet: 1, expected: 0.7 },
+ ])(
+ "preserves stereo energy for phase $phase and channel gain $quiet",
+ async ({ phase, quiet, expected }) => {
+ configure();
+ const rate = 44_100;
+ const left = Float32Array.from(
+ { length: rate },
+ (_, k) => 0.7 * quiet * Math.sin((2 * Math.PI * 440 * k) / rate),
+ );
+ const right = Float32Array.from(
+ { length: rate },
+ (_, k) => 0.7 * Math.sin((2 * Math.PI * 440 * k) / rate + phase),
+ );
+ reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });
+ await analyzeSignal(
+ { sampleRate: rate, channels: [left, right] },
+ { capabilities: ["describe"] },
+ [apiProvider],
+ );
+ const wav = Buffer.from(received[0]!.audio.bytes);
+ expect(wav.readUInt32LE(24)).toBe(UPLOAD_RATE);
+ let energy = 0;
+ for (let k = 0; k < UPLOAD_RATE; k++) energy += (wav.readInt16LE(44 + 2 * k) / 32767) ** 2;
+ expect(Math.sqrt((2 * energy) / UPLOAD_RATE)).toBeCloseTo(expected, 2);
+ },
+ );
+
+ it("reads an answer sent without the data envelope too", async () => {
+ configure();
+ enveloped = false;
+ reply = () => ({ status: 200, body: { version: 1, describe: { text: "bare", tags: [] } } });
+ const analysis = await analyzeSignal(clicks(2), { capabilities: ["describe"] }, [apiProvider]);
+ expect(analysis.values.describe).toEqual({ text: "bare", tags: [] });
+ });
+
+ it("never sends the key anywhere but the x-api-key header", async () => {
configure();
reply = () => ({ status: 401, body: { error: { message: "bad key" } } });
const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]);
expect(JSON.stringify(analysis)).not.toContain("test-key");
+ const [call] = received;
+ expect(call).toMatchObject({ apiKey: "test-key", authorization: undefined });
+ expect(JSON.stringify(call!.request)).not.toContain("test-key");
});
});
diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts
index 5659f19206..8db80a854c 100644
--- a/packages/cli/src/audio/analysis/providers/api.ts
+++ b/packages/cli/src/audio/analysis/providers/api.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
import {
medianInterval,
rhythm,
@@ -12,14 +13,14 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.
/**
* The audio analysis service contract.
*
- * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with
- * `Authorization: Bearer <HYPERFRAMES_AUDIO_API_KEY>` and a `multipart/form-data` body of two parts:
+ * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with `x-api-key: <HYPERFRAMES_AUDIO_API_KEY>`,
+ * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts:
* - `request`: `application/json`, an {@link AudioApiRequest};
- * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded.
+ * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz.
*
- * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the
- * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure;
- * its body may be `{ "error": { "message": string } }`.
+ * Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read
+ * too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left
+ * out of the response. Any other status is a failure; its body may be `{ "error": { "message": string } }`.
*/
export interface AudioApiRequest {
version: 1;
@@ -33,6 +34,8 @@ export interface AudioApiResponse {
version: 1;
/** What answered, as the service wants it reported. */
source?: string;
+ /** The finest step the service's times resolve, in seconds. */
+ time_resolution_seconds?: number;
beats?: { times: number[] };
downbeats?: { times: number[]; meter?: number | null };
sections?: { spans: { start: number; end: number; label?: string }[] };
@@ -43,6 +46,68 @@ const URL_ENV = "HYPERFRAMES_AUDIO_API_URL";
const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";
const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";
const DEFAULT_TIMEOUT_MS = 120_000;
+const UPLOAD_RATE = 16_000;
+const SINC_ZEROS = 16;
+const ROLLOFF = 0.9;
+const SINC_STEPS = 512;
+const HANN_SINC = Float32Array.from({ length: SINC_ZEROS * SINC_STEPS + 2 }, (_, k) => {
+ const x = k / SINC_STEPS;
+ if (x >= SINC_ZEROS) return 0;
+ const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+ return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / SINC_ZEROS));
+});
+
+/** Windowed-sinc resampling with its cutoff just under the lower Nyquist, so nothing above it folds back in. */
+function resample(input: Float32Array, from: number, to: number): Float32Array {
+ if (from === to) return input;
+ const step = from / to;
+ const scale = Math.min(1, to / from) * ROLLOFF;
+ const reach = SINC_ZEROS / scale;
+ const output = new Float32Array(Math.round(input.length / step));
+ for (let i = 0; i < output.length; i++) {
+ const center = i * step;
+ const last = Math.min(input.length - 1, Math.floor(center + reach));
+ let sum = 0;
+ let weights = 0;
+ for (let n = Math.max(0, Math.ceil(center - reach)); n <= last; n++) {
+ const at = Math.abs(n - center) * scale * SINC_STEPS;
+ const k = Math.floor(at);
+ const weight = HANN_SINC[k]! + (at - k) * (HANN_SINC[k + 1]! - HANN_SINC[k]!);
+ sum += input[n]! * weight;
+ weights += weight;
+ }
+ output[i] = weights ? sum / weights : 0;
+ }
+ return output;
+}
+
+/** Preserve a channel when destructive interference removes over 90% of its energy. */
+function monoInput(channels: readonly Float32Array[]): Float32Array {
+ const mono = new Float32Array(channels[0]?.length ?? 0);
+ let loudest: Float32Array = mono;
+ let loudestEnergy = 0;
+ for (const channel of channels) {
+ let energy = 0;
+ for (let i = 0; i < mono.length; i++) {
+ const value = channel[i]!;
+ mono[i]! += value / channels.length;
+ energy += value * value;
+ }
+ if (energy > loudestEnergy) {
+ loudest = channel;
+ loudestEnergy = energy;
+ }
+ }
+ const monoEnergy = mono.reduce((energy, value) => energy + value * value, 0);
+ return monoEnergy < loudestEnergy * 0.1 ? loudest : mono;
+}
+
+function uploaded(signal: AudioSignal): AudioSignal {
+ return {
+ sampleRate: UPLOAD_RATE,
+ channels: [resample(monoInput(signal.channels), signal.sampleRate, UPLOAD_RATE)],
+ };
+}
/** Beats and downbeats come from one request; the others each from their own. */
const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] =>
@@ -90,16 +155,17 @@ async function failure(response: Response): Promise<Error> {
return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);
}
-async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {
+async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) {
const { url, key, timeoutMs } = config();
if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);
+ const signal = uploaded(analyzed);
const request: AudioApiRequest = {
version: 1,
capabilities,
audio: {
sampleRate: signal.sampleRate,
- channels: signal.channels.length,
- duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate,
+ channels: 1,
+ duration: signal.channels[0]!.length / signal.sampleRate,
},
};
const form = new FormData();
@@ -111,12 +177,14 @@ async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {
);
const response = await fetch(url, {
method: "POST",
- headers: { authorization: `Bearer ${key}` },
+ redirect: "error",
+ headers: { "x-api-key": key, "idempotency-key": randomUUID() },
body: form,
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) throw await failure(response);
- return (await response.json()) as AudioApiResponse;
+ const body = (await response.json()) as { data?: AudioApiResponse } | null;
+ return (body?.data ?? body ?? {}) as AudioApiResponse;
}
const requests = new Map<string, (signal: AudioSignal) => Promise<AudioApiResponse>>();