Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(voice): upload segments as bytes, never as a Blob

Samrat Talukdar1850b71691f3
diff --git a/apps/app/src/app/lib/voice-core/index.ts b/apps/app/src/app/lib/voice-core/index.ts
index a5356cca..6f6889fa 100644
--- a/apps/app/src/app/lib/voice-core/index.ts
+++ b/apps/app/src/app/lib/voice-core/index.ts
@@ -16,6 +16,6 @@ export { Downsampler, resampleTo16k } from "./resample";
export { LevelMeter, Segmenter, levelFromRms, rms, type Segment, type SegmenterOptions } from "./segmenter";
export { countWords, isCjk, isLikelyHallucination, padInsertion, stitch } from "./text";
export { DEV_KEYTERMS, applyKeyterms, buildKeyterms, splitIdentifier, type KeytermContext } from "./keyterms";
-export { RemoteTranscriber, errorCode, voiceErrorFromResponse, type RemoteTranscriberOptions } from "./transcriber";
+export { RemoteTranscriber, errorCode, multipartBody, voiceErrorFromResponse, type RemoteTranscriberOptions } from "./transcriber";
export { CircuitBreaker, VoiceSession, type VoiceSessionOptions } from "./session";
export { VOICE_INPUT_FILE_ENV, WavFileSource, voiceInputFileFromEnv } from "./file-source";
diff --git a/apps/app/src/app/lib/voice-core/transcriber.ts b/apps/app/src/app/lib/voice-core/transcriber.ts
index afed30ff..e6400640 100644
--- a/apps/app/src/app/lib/voice-core/transcriber.ts
+++ b/apps/app/src/app/lib/voice-core/transcriber.ts
@@ -69,6 +69,28 @@ export function voiceErrorFromResponse(status: number, code: string | null, retr
});
}
+/**
+ * multipart/form-data as plain bytes. Not FormData with a Blob: a browser
+ * may page Blob contents into its blob store on disk, and audio must only
+ * ever be in memory.
+ */
+export function multipartBody(wav: Uint8Array, filename: string, fields: ReadonlyArray<[string, string]>): { body: Uint8Array<ArrayBuffer>; contentType: string } {
+ const random = new Uint8Array(12);
+ globalThis.crypto.getRandomValues(random);
+ const boundary = `----omnirush-voice-${Array.from(random, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+ const encoder = new TextEncoder();
+ const head = encoder.encode(
+ fields.map(([name, value]) => `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value.replace(/[\r\n]+/g, " ")}\r\n`).join("")
+ + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: audio/wav\r\n\r\n`,
+ );
+ const tail = encoder.encode(`\r\n--${boundary}--\r\n`);
+ const body = new Uint8Array(head.length + wav.length + tail.length);
+ body.set(head, 0);
+ body.set(wav, head.length);
+ body.set(tail, head.length + wav.length);
+ return { body, contentType: `multipart/form-data; boundary=${boundary}` };
+}
+
/**
* A Transcriber over HTTP: one multipart POST per segment (`file`,
* `language`, `prompt_terms`, `segment_index`, `recording_id`), answered
@@ -78,21 +100,18 @@ export class RemoteTranscriber implements Transcriber {
constructor(private readonly options: RemoteTranscriberOptions) {}
async transcribe(request: TranscribeRequest, signal: AbortSignal): Promise<TranscribeResult> {
- const form = new FormData();
- const wav = request.wav;
- form.append("file", new Blob([wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength) as ArrayBuffer], { type: "audio/wav" }), `segment-${request.segmentIndex}.wav`);
- if (request.language) form.append("language", request.language);
- if (request.keyterms.length > 0) form.append("prompt_terms", request.keyterms.join(", "));
- form.append("segment_index", String(request.segmentIndex));
- form.append("recording_id", request.recordingId);
+ const fields: Array<[string, string]> = [["segment_index", String(request.segmentIndex)], ["recording_id", request.recordingId]];
+ if (request.language) fields.push(["language", request.language]);
+ if (request.keyterms.length > 0) fields.push(["prompt_terms", request.keyterms.join(", ")]);
+ const { body: upload, contentType } = multipartBody(request.wav, `segment-${request.segmentIndex}.wav`, fields);
const fetcher = this.options.fetch ?? ((input: string, init: RequestInit) => fetch(input, init));
const timeout = AbortSignal.timeout(this.options.timeoutMs ?? 30_000);
let response: Response;
try {
response = await fetcher(this.options.url, {
method: "POST",
- headers: { Accept: "application/json", ...(await this.options.headers?.()) },
- body: form,
+ headers: { Accept: "application/json", "Content-Type": contentType, ...(await this.options.headers?.()) },
+ body: upload,
signal: AbortSignal.any([signal, timeout]),
});
} catch (error) {
diff --git a/apps/app/tests/voice-core.test.ts b/apps/app/tests/voice-core.test.ts
index 5230b7c1..eef7db2c 100644
--- a/apps/app/tests/voice-core.test.ts
+++ b/apps/app/tests/voice-core.test.ts
@@ -389,27 +389,29 @@ describe("remote transcriber", () => {
};
test("posts a multipart segment and reads the text", async () => {
- let seen: FormData | null = null;
- let headers: Record<string, string> = {};
+ let seen: Request | null = null;
const transcriber = new RemoteTranscriber({
url: "http://127.0.0.1:1/omnirush/voice/transcribe",
headers: () => ({ Authorization: "Bearer t" }),
- fetch: async (_url, init) => {
- seen = init.body as FormData;
- headers = init.headers as Record<string, string>;
+ fetch: async (url, init) => {
+ // Plain bytes, never a Blob (a browser may page Blobs to disk).
+ expect(init.body).toBeInstanceOf(Uint8Array);
+ seen = new Request(url, init);
return Response.json({ text: "hello", provider: "primary", duration_ms: 100 });
},
});
expect(await transcriber.transcribe(request, new AbortController().signal)).toEqual({ text: "hello" });
- const form = seen as unknown as FormData;
+ const upload = seen as unknown as Request;
+ expect(upload.headers.get("authorization")).toBe("Bearer t");
+ const form = await upload.formData();
expect(form.get("segment_index")).toBe("2");
expect(form.get("recording_id")).toBe("abc");
expect(form.get("language")).toBe("en");
expect(form.get("prompt_terms")).toBe("OmniRush, voice");
const file = form.get("file") as File;
- expect(file.type).toBe("audio/wav");
+ expect(file.type).toMatch(/wav/);
expect(file.size).toBe(request.wav.length);
- expect(headers.Authorization).toBe("Bearer t");
+ expect(new Uint8Array(await file.arrayBuffer())).toEqual(request.wav);
});
test.each([
diff --git a/apps/app/tests/voice-dictation-ui.test.tsx b/apps/app/tests/voice-dictation-ui.test.tsx
index df62fe5a..f807e205 100644
--- a/apps/app/tests/voice-dictation-ui.test.tsx
+++ b/apps/app/tests/voice-dictation-ui.test.tsx
@@ -116,7 +116,7 @@ describe("dictation in the composer", () => {
const fetchStub = async (input: RequestInfo | URL, init?: RequestInit) => {
const url = String(input);
if (!url.endsWith("/omnirush/voice/transcribe")) return new Response("{}", { status: 404 });
- const form = init?.body as FormData;
+ const form = await new Request(url, init).formData();
uploads.push(form);
await new Promise((resolve) => setTimeout(resolve, 30));
return answers(Number(form.get("segment_index")));