Skip to content

Write HTML. Render video. Built for agents.

fix(cli): analyze finalized audio assets after direct upload (#5647)

Xuanru Li34776f3def03
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index fb4aea133b..42026315d1 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -567,7 +567,9 @@ answers:
- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and
`HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono
- WAV. Channels are averaged unless cancellation removes over 90% of the loudest
+ WAV through the same origin’s public asset direct-upload and completion endpoints.
+ Each decoded signal is uploaded once; analysis requests send its finalized asset ID as JSON.
+ The API key needs asset upload and audio-analysis permissions. Channels are averaged unless cancellation removes over 90% of the loudest
channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each
request; the answer may come wrapped in `{ "data": … }`. Every beat,
downbeat or section boundary it answers moves onto the nearest beat signal
diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts
index 2fb5d919c2..09f5067ce5 100644
--- a/packages/cli/src/audio/analysis/providers/api.test.ts
+++ b/packages/cli/src/audio/analysis/providers/api.test.ts
@@ -42,25 +42,56 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = ()
body: {},
});
let enveloped = true;
+let uploads = 0;
+let uploadedBytes = new Uint8Array();
-async function readForm(req: IncomingMessage) {
- const chunks: Buffer[] = [];
- for await (const chunk of req) chunks.push(chunk as Buffer);
- const form = await new Request("http://stand-in", {
- method: "POST",
- headers: { "content-type": req.headers["content-type"]! },
- body: Buffer.concat(chunks),
- }).formData();
- const audio = form.get("audio") as Blob;
- return {
- request: JSON.parse(await (form.get("request") as Blob).text()) as AudioApiRequest,
- audio: { type: audio.type, bytes: new Uint8Array(await audio.arrayBuffer()) },
- };
+async function readBytes(req: IncomingMessage) {
+ const chunks: Uint8Array[] = [];
+ for await (const chunk of req) chunks.push(chunk);
+ return Buffer.concat(chunks);
}
beforeAll(async () => {
server = createServer(async (req, res) => {
- const form = await readForm(req);
+ const bytes = await readBytes(req);
+ if (req.url === "/v3/assets/direct-uploads") {
+ expect(req.headers["x-api-key"]).toBe("test-key");
+ expect(req.headers.authorization).toBeUndefined();
+ const input = JSON.parse(bytes.toString());
+ expect(input).toMatchObject({ filename: "audio.wav", content_type: "audio/wav" });
+ uploads++;
+ res.writeHead(200, { "content-type": "application/json" }).end(
+ JSON.stringify({
+ data: {
+ asset_id: "uploaded-audio",
+ upload_url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/put`,
+ upload_headers: { "content-type": "audio/wav" },
+ },
+ }),
+ );
+ return;
+ }
+ if (req.url === "/put") {
+ expect(req.method).toBe("PUT");
+ expect(req.headers["x-api-key"]).toBeUndefined();
+ expect(req.headers.authorization).toBeUndefined();
+ expect(req.headers["content-type"]).toBe("audio/wav");
+ uploadedBytes = new Uint8Array(bytes);
+ res.writeHead(200).end();
+ return;
+ }
+ if (req.url === "/v3/assets/uploaded-audio/complete") {
+ expect(req.headers["x-api-key"]).toBe("test-key");
+ res
+ .writeHead(200, { "content-type": "application/json" })
+ .end(JSON.stringify({ data: { asset_id: "uploaded-audio" } }));
+ return;
+ }
+ expect(req.headers["content-type"]).toBe("application/json");
+ const form = {
+ request: JSON.parse(bytes.toString()) as AudioApiRequest,
+ audio: { type: "audio/wav", bytes: uploadedBytes },
+ };
received.push({
method: req.method!,
apiKey: req.headers["x-api-key"] as string | undefined,
@@ -79,6 +110,7 @@ afterAll(() => new Promise<void>((done) => server.close(() => done())));
afterEach(() => {
received = [];
+ uploads = 0;
enveloped = true;
delete process.env.HYPERFRAMES_AUDIO_API_URL;
delete process.env.HYPERFRAMES_AUDIO_API_KEY;
@@ -162,7 +194,7 @@ describe("apiProvider", () => {
expect(call!.request).toEqual({
version: 1,
capabilities: ["beats", "downbeats"],
- audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 },
+ audio: { type: "asset_id", asset_id: "uploaded-audio" },
});
expect(call!.audio.type).toBe("audio/wav");
expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");
@@ -246,6 +278,8 @@ describe("apiProvider", () => {
]);
expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);
expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2);
+ expect(uploads).toBe(1);
+ expect(received.every((r) => r.request.audio.asset_id === "uploaded-audio")).toBe(true);
expect(analysis.values.sections).toMatchObject({
spans: [
{ start: 0, end: beat, label: "intro" },
diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts
index 8db80a854c..29451eccc8 100644
--- a/packages/cli/src/audio/analysis/providers/api.ts
+++ b/packages/cli/src/audio/analysis/providers/api.ts
@@ -1,3 +1,5 @@
+import { HyperframesCloudClient } from "../../../cloud/_gen/client.js";
+import { uploadAssetViaDirectUpload } from "../../../cloud/upload.js";
import { randomUUID } from "node:crypto";
import {
medianInterval,
@@ -14,9 +16,9 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.
* The audio analysis service contract.
*
* Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with `x-api-key: <HYPERFRAMES_AUDIO_API_KEY>`,
- * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts:
- * - `request`: `application/json`, an {@link AudioApiRequest};
- * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz.
+ * a new `Idempotency-Key` per request, and an {@link AudioApiRequest} JSON body.
+ * The analyzed span is uploaded once as 16-bit PCM WAV, mono at 16 kHz, using the same origin
+ * public asset direct-upload and completion routes. The body references that finalized asset.
*
* Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read
* too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left
@@ -25,7 +27,7 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.
export interface AudioApiRequest {
version: 1;
capabilities: AudioApiCapability[];
- audio: { sampleRate: number; channels: number; duration: number };
+ audio: { type: "asset_id"; asset_id: string };
}
export type AudioApiCapability = "beats" | "downbeats" | "sections" | "describe";
@@ -155,31 +157,49 @@ async function failure(response: Response): Promise<Error> {
return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);
}
+const uploadedAsset = perSignal(async (analyzed: AudioSignal) => {
+ const { url, key, timeoutMs } = config();
+ if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);
+ const deadline = AbortSignal.timeout(timeoutMs);
+ const boundedFetch: typeof fetch = (input, init) =>
+ fetch(input, {
+ ...init,
+ redirect: "error",
+ signal: deadline,
+ });
+ const client = new HyperframesCloudClient({
+ baseUrl: new URL(url).origin,
+ getAuthHeaders: () => ({ "x-api-key": key }),
+ fetchImpl: boundedFetch,
+ });
+ const asset = await uploadAssetViaDirectUpload({
+ client,
+ bytes: new Uint8Array(encodeWav(uploaded(analyzed))),
+ filename: "audio.wav",
+ contentType: "audio/wav",
+ idempotencyKey: randomUUID(),
+ fetchImpl: boundedFetch,
+ });
+ return asset.asset_id;
+});
+
async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) {
const { url, key, timeoutMs } = config();
if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);
- const signal = uploaded(analyzed);
const request: AudioApiRequest = {
version: 1,
capabilities,
- audio: {
- sampleRate: signal.sampleRate,
- channels: 1,
- duration: signal.channels[0]!.length / signal.sampleRate,
- },
+ audio: { type: "asset_id", asset_id: await uploadedAsset(analyzed) },
};
- const form = new FormData();
- form.append("request", new Blob([JSON.stringify(request)], { type: "application/json" }));
- form.append(
- "audio",
- new Blob([new Uint8Array(encodeWav(signal))], { type: "audio/wav" }),
- "audio.wav",
- );
const response = await fetch(url, {
method: "POST",
redirect: "error",
- headers: { "x-api-key": key, "idempotency-key": randomUUID() },
- body: form,
+ headers: {
+ "x-api-key": key,
+ "idempotency-key": randomUUID(),
+ "content-type": "application/json",
+ },
+ body: JSON.stringify(request),
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) throw await failure(response);
diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.ts
index 51c4466f0d..703b83e697 100644
--- a/packages/cli/src/cloud/upload.ts
+++ b/packages/cli/src/cloud/upload.ts
@@ -85,9 +85,10 @@ async function putBytesToPresignedUrl(
uploadUrl: string,
uploadHeaders: Record<string, unknown>,
bytes: Uint8Array,
+ contentType: string,
): Promise<void> {
const headers: Record<string, string> = {
- "content-type": CONTENT_TYPE_ZIP,
+ "content-type": contentType,
...normalizeUploadHeaders(uploadHeaders),
};
// `Uint8Array<ArrayBufferLike>` is a valid `BodyInit` at runtime but
@@ -136,8 +137,8 @@ async function completeWithRetry(
throw lastErr ?? new Error("completeAssetUpload retries exhausted");
}
-export async function uploadZipViaDirectUpload(
- opts: UploadZipViaDirectOptions,
+export async function uploadAssetViaDirectUpload(
+ opts: UploadZipViaDirectOptions & { contentType: string },
): Promise<UploadZipViaDirectResult> {
const start = Date.now();
const { bytes, filename, idempotencyKey } = opts;
@@ -147,12 +148,18 @@ export async function uploadZipViaDirectUpload(
opts.onProgress?.({ phase: "initialize" });
const initialize = await opts.client.createAssetUpload({
- body: { filename, content_type: CONTENT_TYPE_ZIP, size_bytes, checksum_sha256 },
+ body: { filename, content_type: opts.contentType, size_bytes, checksum_sha256 },
idempotencyKey,
});
opts.onProgress?.({ phase: "upload", percent: 0 });
- await putBytesToPresignedUrl(fetchImpl, initialize.upload_url, initialize.upload_headers, bytes);
+ await putBytesToPresignedUrl(
+ fetchImpl,
+ initialize.upload_url,
+ initialize.upload_headers,
+ bytes,
+ opts.contentType,
+ );
opts.onProgress?.({ phase: "upload", percent: 100 });
opts.onProgress?.({ phase: "complete" });
@@ -163,3 +170,9 @@ export async function uploadZipViaDirectUpload(
duration_ms: Date.now() - start,
};
}
+
+export function uploadZipViaDirectUpload(
+ opts: UploadZipViaDirectOptions,
+): Promise<UploadZipViaDirectResult> {
+ return uploadAssetViaDirectUpload({ ...opts, contentType: CONTENT_TYPE_ZIP });
+}