Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): analyze finalized audio assets after direct upload (#5647)
Xuanru Li34776f3def03
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex fb4aea133b..42026315d1 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -567,7 +567,9 @@ answers:- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and`HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono- WAV. Channels are averaged unless cancellation removes over 90% of the loudest+ WAV through the same origin’s public asset direct-upload and completion endpoints.+ Each decoded signal is uploaded once; analysis requests send its finalized asset ID as JSON.+ The API key needs asset upload and audio-analysis permissions. Channels are averaged unless cancellation removes over 90% of the loudestchannel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on eachrequest; the answer may come wrapped in `{ "data": … }`. Every beat,downbeat or section boundary it answers moves onto the nearest beat signaldiff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.tsindex 2fb5d919c2..09f5067ce5 100644--- a/packages/cli/src/audio/analysis/providers/api.test.ts+++ b/packages/cli/src/audio/analysis/providers/api.test.ts@@ -42,25 +42,56 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = ()body: {},});let enveloped = true;+let uploads = 0;+let uploadedBytes = new Uint8Array();-async function readForm(req: IncomingMessage) {- const chunks: Buffer[] = [];- for await (const chunk of req) chunks.push(chunk as Buffer);- const form = await new Request("http://stand-in", {- method: "POST",- headers: { "content-type": req.headers["content-type"]! },- body: Buffer.concat(chunks),- }).formData();- const audio = form.get("audio") as Blob;- return {- request: JSON.parse(await (form.get("request") as Blob).text()) as AudioApiRequest,- audio: { type: audio.type, bytes: new Uint8Array(await audio.arrayBuffer()) },- };+async function readBytes(req: IncomingMessage) {+ const chunks: Uint8Array[] = [];+ for await (const chunk of req) chunks.push(chunk);+ return Buffer.concat(chunks);}beforeAll(async () => {server = createServer(async (req, res) => {- const form = await readForm(req);+ const bytes = await readBytes(req);+ if (req.url === "/v3/assets/direct-uploads") {+ expect(req.headers["x-api-key"]).toBe("test-key");+ expect(req.headers.authorization).toBeUndefined();+ const input = JSON.parse(bytes.toString());+ expect(input).toMatchObject({ filename: "audio.wav", content_type: "audio/wav" });+ uploads++;+ res.writeHead(200, { "content-type": "application/json" }).end(+ JSON.stringify({+ data: {+ asset_id: "uploaded-audio",+ upload_url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/put`,+ upload_headers: { "content-type": "audio/wav" },+ },+ }),+ );+ return;+ }+ if (req.url === "/put") {+ expect(req.method).toBe("PUT");+ expect(req.headers["x-api-key"]).toBeUndefined();+ expect(req.headers.authorization).toBeUndefined();+ expect(req.headers["content-type"]).toBe("audio/wav");+ uploadedBytes = new Uint8Array(bytes);+ res.writeHead(200).end();+ return;+ }+ if (req.url === "/v3/assets/uploaded-audio/complete") {+ expect(req.headers["x-api-key"]).toBe("test-key");+ res+ .writeHead(200, { "content-type": "application/json" })+ .end(JSON.stringify({ data: { asset_id: "uploaded-audio" } }));+ return;+ }+ expect(req.headers["content-type"]).toBe("application/json");+ const form = {+ request: JSON.parse(bytes.toString()) as AudioApiRequest,+ audio: { type: "audio/wav", bytes: uploadedBytes },+ };received.push({method: req.method!,apiKey: req.headers["x-api-key"] as string | undefined,@@ -79,6 +110,7 @@ afterAll(() => new Promise<void>((done) => server.close(() => done())));afterEach(() => {received = [];+ uploads = 0;enveloped = true;delete process.env.HYPERFRAMES_AUDIO_API_URL;delete process.env.HYPERFRAMES_AUDIO_API_KEY;@@ -162,7 +194,7 @@ describe("apiProvider", () => {expect(call!.request).toEqual({version: 1,capabilities: ["beats", "downbeats"],- audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 },+ audio: { type: "asset_id", asset_id: "uploaded-audio" },});expect(call!.audio.type).toBe("audio/wav");expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");@@ -246,6 +278,8 @@ describe("apiProvider", () => {]);expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2);+ expect(uploads).toBe(1);+ expect(received.every((r) => r.request.audio.asset_id === "uploaded-audio")).toBe(true);expect(analysis.values.sections).toMatchObject({spans: [{ start: 0, end: beat, label: "intro" },diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.tsindex 8db80a854c..29451eccc8 100644--- a/packages/cli/src/audio/analysis/providers/api.ts+++ b/packages/cli/src/audio/analysis/providers/api.ts@@ -1,3 +1,5 @@+import { HyperframesCloudClient } from "../../../cloud/_gen/client.js";+import { uploadAssetViaDirectUpload } from "../../../cloud/upload.js";import { randomUUID } from "node:crypto";import {medianInterval,@@ -14,9 +16,9 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.* The audio analysis service contract.** Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with `x-api-key: <HYPERFRAMES_AUDIO_API_KEY>`,- * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts:- * - `request`: `application/json`, an {@link AudioApiRequest};- * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz.+ * a new `Idempotency-Key` per request, and an {@link AudioApiRequest} JSON body.+ * The analyzed span is uploaded once as 16-bit PCM WAV, mono at 16 kHz, using the same origin+ * public asset direct-upload and completion routes. The body references that finalized asset.** Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read* too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left@@ -25,7 +27,7 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.export interface AudioApiRequest {version: 1;capabilities: AudioApiCapability[];- audio: { sampleRate: number; channels: number; duration: number };+ audio: { type: "asset_id"; asset_id: string };}export type AudioApiCapability = "beats" | "downbeats" | "sections" | "describe";@@ -155,31 +157,49 @@ async function failure(response: Response): Promise<Error> {return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);}+const uploadedAsset = perSignal(async (analyzed: AudioSignal) => {+ const { url, key, timeoutMs } = config();+ if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);+ const deadline = AbortSignal.timeout(timeoutMs);+ const boundedFetch: typeof fetch = (input, init) =>+ fetch(input, {+ ...init,+ redirect: "error",+ signal: deadline,+ });+ const client = new HyperframesCloudClient({+ baseUrl: new URL(url).origin,+ getAuthHeaders: () => ({ "x-api-key": key }),+ fetchImpl: boundedFetch,+ });+ const asset = await uploadAssetViaDirectUpload({+ client,+ bytes: new Uint8Array(encodeWav(uploaded(analyzed))),+ filename: "audio.wav",+ contentType: "audio/wav",+ idempotencyKey: randomUUID(),+ fetchImpl: boundedFetch,+ });+ return asset.asset_id;+});+async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) {const { url, key, timeoutMs } = config();if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);- const signal = uploaded(analyzed);const request: AudioApiRequest = {version: 1,capabilities,- audio: {- sampleRate: signal.sampleRate,- channels: 1,- duration: signal.channels[0]!.length / signal.sampleRate,- },+ audio: { type: "asset_id", asset_id: await uploadedAsset(analyzed) },};- const form = new FormData();- form.append("request", new Blob([JSON.stringify(request)], { type: "application/json" }));- form.append(- "audio",- new Blob([new Uint8Array(encodeWav(signal))], { type: "audio/wav" }),- "audio.wav",- );const response = await fetch(url, {method: "POST",redirect: "error",- headers: { "x-api-key": key, "idempotency-key": randomUUID() },- body: form,+ headers: {+ "x-api-key": key,+ "idempotency-key": randomUUID(),+ "content-type": "application/json",+ },+ body: JSON.stringify(request),signal: AbortSignal.timeout(timeoutMs),});if (!response.ok) throw await failure(response);diff --git a/packages/cli/src/cloud/upload.ts b/packages/cli/src/cloud/upload.tsindex 51c4466f0d..703b83e697 100644--- a/packages/cli/src/cloud/upload.ts+++ b/packages/cli/src/cloud/upload.ts@@ -85,9 +85,10 @@ async function putBytesToPresignedUrl(uploadUrl: string,uploadHeaders: Record<string, unknown>,bytes: Uint8Array,+ contentType: string,): Promise<void> {const headers: Record<string, string> = {- "content-type": CONTENT_TYPE_ZIP,+ "content-type": contentType,...normalizeUploadHeaders(uploadHeaders),};// `Uint8Array<ArrayBufferLike>` is a valid `BodyInit` at runtime but@@ -136,8 +137,8 @@ async function completeWithRetry(throw lastErr ?? new Error("completeAssetUpload retries exhausted");}-export async function uploadZipViaDirectUpload(- opts: UploadZipViaDirectOptions,+export async function uploadAssetViaDirectUpload(+ opts: UploadZipViaDirectOptions & { contentType: string },): Promise<UploadZipViaDirectResult> {const start = Date.now();const { bytes, filename, idempotencyKey } = opts;@@ -147,12 +148,18 @@ export async function uploadZipViaDirectUpload(opts.onProgress?.({ phase: "initialize" });const initialize = await opts.client.createAssetUpload({- body: { filename, content_type: CONTENT_TYPE_ZIP, size_bytes, checksum_sha256 },+ body: { filename, content_type: opts.contentType, size_bytes, checksum_sha256 },idempotencyKey,});opts.onProgress?.({ phase: "upload", percent: 0 });- await putBytesToPresignedUrl(fetchImpl, initialize.upload_url, initialize.upload_headers, bytes);+ await putBytesToPresignedUrl(+ fetchImpl,+ initialize.upload_url,+ initialize.upload_headers,+ bytes,+ opts.contentType,+ );opts.onProgress?.({ phase: "upload", percent: 100 });opts.onProgress?.({ phase: "complete" });@@ -163,3 +170,9 @@ export async function uploadZipViaDirectUpload(duration_ms: Date.now() - start,};}++export function uploadZipViaDirectUpload(+ opts: UploadZipViaDirectOptions,+): Promise<UploadZipViaDirectResult> {+ return uploadAssetViaDirectUpload({ ...opts, contentType: CONTENT_TYPE_ZIP });+}