Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Use OpenAI limits for voice instead of our own caps (#2259)

Regan Bellba02a66eb519
diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.ts
index 89f18fee5..31b317607 100644
--- a/plugins/web-ui/server/index.ts
+++ b/plugins/web-ui/server/index.ts
@@ -1,5 +1,5 @@
import "./instrument.ts";
-import { handleVoiceRoute } from "./voice-routes.ts";
+import { handleVoiceRoute, sweepDictationSpool } from "./voice-routes.ts";
import { browserErrorConfig } from "./browser-error-config.ts";
import { flushErrorReporting, reportBackendError } from "../../chassis/src/error-reporting.ts";
import { appEditSlug } from "../src/app-edit.ts";
@@ -3304,8 +3304,6 @@ const routeRequest = async (req: IncomingMessage, res: ServerResponse) => {
if (!user) return unauthorized(res, req);
if (["/api/dictation", "/api/voice", "/api/voice/sessions"].includes(path)) {
return handleVoiceRoute(req, res, path, resolveIdentity(req)!, {
- core: CORE,
- secret: CORE_SIGNING_SECRET,
org: ORG,
publicUrl: PUBLIC_URL,
});
@@ -3424,6 +3422,7 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
t.unref?.();
void runStateFeed();
void runInboxFeed();
+ void sweepDictationSpool();
});
})
.catch(async (err: unknown) => {
diff --git a/plugins/web-ui/server/voice-routes.ts b/plugins/web-ui/server/voice-routes.ts
index c17779e94..4bd491c85 100644
--- a/plugins/web-ui/server/voice-routes.ts
+++ b/plugins/web-ui/server/voice-routes.ts
@@ -1,13 +1,17 @@
+import { randomUUID } from "node:crypto";
+import fs from "node:fs";
+import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
import type { IncomingMessage, ServerResponse } from "node:http";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pipeline } from "node:stream/promises";
import { json, readBodyBytes, PayloadTooLargeError } from "../../chassis/src/http.ts";
-import { coreClaimStore, withinRateLimit } from "../../chassis/src/claims.ts";
import { reportBackendError } from "../../chassis/src/error-reporting.ts";
import { errChain } from "../../chassis/src/errors.ts";
import {
createLiveVoice,
dictationFormats,
LIVE_MODEL,
- MAX_DICTATION_BYTES,
openaiKey,
transcribe,
voiceEnabled,
@@ -15,7 +19,36 @@ import {
VoiceProviderError,
} from "./voice.ts";
-const BODY_READ_TIMEOUT_MS = 30_000;
+const TRANSCRIBE_TIMEOUT_MS = 600_000;
+const LIVE_SESSION_TIMEOUT_MS = 25_000;
+const MAX_RETRY_AFTER_S = 3600;
+const REJECTED_STATUSES = new Set([400, 401, 403, 404, 422]);
+
+const SPOOL_DIR = join(tmpdir(), "qm-dictation");
+const STALE_SPOOL_MS = 3_600_000;
+
+export async function sweepDictationSpool(nowMs = Date.now()): Promise<void> {
+ const names = await readdir(SPOOL_DIR).catch(() => []);
+ for (const name of names) {
+ const file = join(SPOOL_DIR, name);
+ const info = await stat(file).catch(() => null);
+ if (info && nowMs - info.mtimeMs > STALE_SPOOL_MS) await rm(file, { force: true }).catch(() => undefined);
+ }
+}
+
+async function spool(req: IncomingMessage, signal: AbortSignal): Promise<{ file: string; size: number }> {
+ await mkdir(SPOOL_DIR, { recursive: true, mode: 0o700 });
+ await chmod(SPOOL_DIR, 0o700);
+ const file = join(SPOOL_DIR, randomUUID());
+ const out = fs.createWriteStream(file, { mode: 0o600 });
+ try {
+ await pipeline(req, out, { signal });
+ } catch (error) {
+ await rm(file, { force: true }).catch(() => undefined);
+ throw error;
+ }
+ return { file, size: out.bytesWritten };
+}
function reportVoiceError(error: unknown, code: string, aborted = false): void {
const detail = { ...(error instanceof VoiceProviderError ? error.detail : {}), aborted };
@@ -28,7 +61,7 @@ export async function handleVoiceRoute(
res: ServerResponse,
path: string,
identity: { user: string; impersonator: string | null; portalVerified: boolean },
- config: { core: string; secret: string | undefined; org: string; publicUrl: string },
+ config: { org: string; publicUrl: string },
): Promise<void> {
const { user } = identity;
const enabled = voiceEnabled(user) && !identity.impersonator;
@@ -66,57 +99,60 @@ export async function handleVoiceRoute(
}
const controller = new AbortController();
+ const finished = new AbortController();
const disconnect = () => {
if (!res.writableEnded) controller.abort();
};
res.once("close", disconnect);
try {
- let allowed: boolean;
- try {
- allowed = await withinRateLimit(coreClaimStore(config.core, config.secret, dictation ? "dictation" : "voice"), {
- secret: config.secret!,
- kind: dictation ? "dictation" : "voice-session",
- value: `${config.org}:${user}`,
- limit: dictation ? 60 : 6,
- windowS: 3600,
- nowMs: Date.now(),
- });
- } catch (error) {
- reportVoiceError(error, dictation ? "dictation_rate_limit_store" : "voice_rate_limit_store");
- res.setHeader("retry-after", "30");
- return json(res, 503, { error: "rate_limit_unavailable", message: "Voice is temporarily unavailable." });
- }
- if (!allowed)
- return dictation
- ? json(res, 429, { error: "dictation_rate_limit", message: "Dictation limit reached. Try again next hour." })
- : json(res, 429, { error: "voice_rate_limit", message: "Voice session limit reached. Try again next hour." });
-
- const caller = {
+ const caller = (timeoutMs: number) => ({
user,
org: config.org,
apiKey: openaiKey()!,
- signal: AbortSignal.any([controller.signal, AbortSignal.timeout(dictation ? 60_000 : 25_000)]),
- };
- if (!dictation) return json(res, 201, await createLiveVoice(sdp, caller));
- const readTimer = setTimeout(() => req.destroy(), BODY_READ_TIMEOUT_MS);
- let audio: Buffer;
+ signal: AbortSignal.any([controller.signal, finished.signal, AbortSignal.timeout(timeoutMs)]),
+ });
+ if (!dictation) return json(res, 201, await createLiveVoice(sdp, caller(LIVE_SESSION_TIMEOUT_MS)));
+ let audio: { file: string; size: number };
try {
- audio = await readBodyBytes(req, MAX_DICTATION_BYTES);
+ audio = await spool(req, controller.signal);
} catch (error) {
- if (error instanceof PayloadTooLargeError) {
- res.setHeader("connection", "close");
- return json(res, 413, { error: "audio_too_large" });
- }
- if (!controller.signal.aborted) reportVoiceError(error, "dictation_upload");
- return controller.signal.aborted ? undefined : json(res, 400, { error: "upload_failed" });
+ const clientGone = controller.signal.aborted;
+ reportVoiceError(error, "dictation_upload", clientGone);
+ if (clientGone) return;
+ return json(res, 503, { error: "dictation_unavailable", message: "The server could not store the recording." });
+ }
+ let text: string | null = null;
+ try {
+ if (audio.size) text = await transcribe(audio.file, audio.size, mimeType, caller(TRANSCRIBE_TIMEOUT_MS));
} finally {
- clearTimeout(readTimer);
+ await rm(audio.file, { force: true }).catch((error: unknown) => reportVoiceError(error, "dictation_cleanup"));
}
- if (!audio.byteLength) return json(res, 400, { error: "empty_audio" });
- return json(res, 200, { text: await transcribe(audio, mimeType, caller) });
+ return text === null ? json(res, 400, { error: "empty_audio" }) : json(res, 200, { text });
} catch (error) {
reportVoiceError(error, dictation ? "dictation_provider" : "voice_provider", controller.signal.aborted);
if (controller.signal.aborted) return;
+ const provider = error instanceof VoiceProviderError ? error : undefined;
+ if (provider?.status === 429 && provider.code !== "insufficient_quota") {
+ if (provider.retryAfterS !== undefined)
+ res.setHeader("retry-after", String(Math.min(provider.retryAfterS, MAX_RETRY_AFTER_S)));
+ return json(res, 429, {
+ error: "provider_rate_limit",
+ message: "OpenAI is rate limiting requests. Try again shortly.",
+ });
+ }
+ if (dictation && provider?.status === 413)
+ return json(res, 413, { error: "audio_too_large", message: "OpenAI says this recording is too large." });
+ if (
+ dictation &&
+ provider?.status === 400 &&
+ (provider.code === "unsupported_format" || (provider.code === "invalid_value" && provider.param === "file"))
+ )
+ return json(res, 400, { error: "invalid_audio", message: "OpenAI could not read this recording." });
+ if (provider?.status !== undefined && REJECTED_STATUSES.has(provider.status))
+ return json(res, 502, {
+ error: dictation ? "dictation_rejected" : "voice_rejected",
+ message: "OpenAI rejected the request. Retrying will not help.",
+ });
return dictation
? json(res, 503, {
error: "dictation_unavailable",
@@ -124,6 +160,7 @@ export async function handleVoiceRoute(
})
: json(res, 503, { error: "voice_unavailable", message: "Voice could not connect. Please try again." });
} finally {
+ finished.abort();
res.off("close", disconnect);
}
}
diff --git a/plugins/web-ui/server/voice.ts b/plugins/web-ui/server/voice.ts
index a0d17b625..595b2fbfe 100644
--- a/plugins/web-ui/server/voice.ts
+++ b/plugins/web-ui/server/voice.ts
@@ -1,11 +1,12 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { Readable } from "node:stream";
import type { IncomingMessage } from "node:http";
import { sameOriginRequest } from "../../chassis/src/http.ts";
import { principalInAllowlist } from "../../chassis/src/principal-allowlist.ts";
export const LIVE_MODEL = "gpt-live-1";
const TRANSCRIBE_MODEL = "gpt-transcribe";
-export const MAX_DICTATION_BYTES = 10 * 1024 * 1024;
export const dictationFormats: Record<string, string> = {
"audio/mp4": "m4a",
"audio/m4a": "m4a",
@@ -16,13 +17,38 @@ export const dictationFormats: Record<string, string> = {
export class VoiceProviderError extends Error {
readonly detail: Record<string, unknown>;
+ readonly status: number | undefined;
+ readonly code: string | undefined;
+ readonly param: string | undefined;
+ readonly retryAfterS: number | undefined;
constructor(message: string, detail: Record<string, unknown>, cause?: unknown) {
super(message, { cause });
this.name = "VoiceProviderError";
this.detail = detail;
+ this.status = typeof detail.status === "number" ? detail.status : undefined;
+ this.code = typeof detail.code === "string" ? detail.code : undefined;
+ this.param = typeof detail.param === "string" ? detail.param : undefined;
+ this.retryAfterS = wholeSeconds(detail.retryAfter);
}
}
+function providerError(body: string): { code?: string; param?: string } {
+ try {
+ const error = (JSON.parse(body) as { error?: { code?: unknown; param?: unknown } } | null)?.error;
+ return {
+ ...(typeof error?.code === "string" ? { code: error.code } : {}),
+ ...(typeof error?.param === "string" ? { param: error.param } : {}),
+ };
+ } catch {
+ return {};
+ }
+}
+
+function wholeSeconds(value: unknown): number | undefined {
+ if (typeof value !== "string" || !value || [...value].some((c) => c < "0" || c > "9")) return undefined;
+ return Number(value);
+}
+
export function openaiKey(env: NodeJS.ProcessEnv = process.env): string | undefined {
return env.OPENAI_API_KEY?.trim() || undefined;
}
@@ -51,12 +77,12 @@ interface Caller {
signal: AbortSignal;
}
-async function openai(path: string, body: FormData | string, headers: Record<string, string>, caller: Caller) {
+async function openai(path: string, body: ReadableStream | string, headers: Record<string, string>, caller: Caller) {
let response: Response;
try {
response = await fetch(`https://api.openai.com/v1/${path}`, {
method: "POST",
- redirect: "error",
+ redirect: "manual",
signal: caller.signal,
headers: {
...headers,
@@ -66,7 +92,8 @@ async function openai(path: string, body: FormData | string, headers: Record<str
.digest("hex"),
},
body,
- });
+ duplex: "half",
+ } as RequestInit & { duplex: "half" });
} catch (error) {
throw new VoiceProviderError(`OpenAI ${path} request failed`, { path }, error);
}
@@ -85,6 +112,7 @@ async function openai(path: string, body: FormData | string, headers: Record<str
if (!response.ok)
throw new VoiceProviderError(`OpenAI ${path} returned HTTP ${response.status}`, {
...detail,
+ ...providerError(text),
body: text.slice(0, 500),
});
try {
@@ -98,20 +126,34 @@ async function openai(path: string, body: FormData | string, headers: Record<str
}
}
-export async function transcribe(audio: Uint8Array, mimeType: string, caller: Caller): Promise<string> {
- const form = new FormData();
- form.set("model", TRANSCRIBE_MODEL);
- form.set(
- "file",
- new Blob([audio as Uint8Array<ArrayBuffer>], { type: mimeType }),
- `dictation.${dictationFormats[mimeType]}`,
+export async function transcribe(file: string, size: number, mimeType: string, caller: Caller): Promise<string> {
+ const boundary = `qm-${randomUUID()}`;
+ const head = Buffer.from(
+ `--${boundary}\r\ncontent-disposition: form-data; name="model"\r\n\r\n${TRANSCRIBE_MODEL}\r\n` +
+ `--${boundary}\r\ncontent-disposition: form-data; name="file"; filename="dictation.${dictationFormats[mimeType]}"\r\n` +
+ `content-type: ${mimeType}\r\n\r\n`,
);
- const data = await openai("audio/transcriptions", form, {}, caller);
- if (typeof data?.text !== "string" || data.text.length > 100_000)
- throw new VoiceProviderError("Invalid transcription response", {
- textType: typeof data?.text,
- length: typeof data?.text === "string" ? data.text.length : undefined,
- });
+ const tail = Buffer.from(`\r\n--${boundary}--\r\n`);
+ const body = Readable.toWeb(
+ Readable.from(
+ (async function* () {
+ yield head;
+ yield* createReadStream(file);
+ yield tail;
+ })(),
+ ),
+ ) as ReadableStream;
+ const data = await openai(
+ "audio/transcriptions",
+ body,
+ {
+ "content-type": `multipart/form-data; boundary=${boundary}`,
+ "content-length": String(head.length + size + tail.length),
+ },
+ caller,
+ );
+ if (typeof data?.text !== "string")
+ throw new VoiceProviderError("Invalid transcription response", { textType: typeof data?.text });
return data.text.trim();
}
diff --git a/plugins/web-ui/test/live-voice-route.test.ts b/plugins/web-ui/test/live-voice-route.test.ts
index 32854b343..e49587d83 100644
--- a/plugins/web-ui/test/live-voice-route.test.ts
+++ b/plugins/web-ui/test/live-voice-route.test.ts
@@ -4,29 +4,6 @@ import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";
-let claimStatus = 200;
-let claimAvailable = true;
-let claims = 0;
-const core = createServer((req, res) => {
- if (!req.url?.startsWith("/v1/auth/broker/claim")) {
- res.writeHead(200, { "content-type": "application/json" });
- res.end("{}");
- return;
- }
- let body = "";
- req.on("data", (chunk) => {
- body += chunk;
- });
- req.on("end", () => {
- claims++;
- const input = JSON.parse(body);
- assert.equal(input.ids.length, 6);
- res.writeHead(claimStatus, { "content-type": "application/json" });
- res.end(JSON.stringify({ claimed: claimAvailable ? input.ids[0] : null }));
- });
-});
-await new Promise<void>((resolve) => core.listen(0, resolve));
-process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;
process.env.CORE_SIGNING_SECRET = "voice-route-test";
process.env.WEB_UI_PUBLIC_URL = "https://qm.example.test";
process.env.WEB_UI_VOICE_USERS = "alice";
@@ -41,7 +18,6 @@ const identity = (user = "alice") => ({
const headers = { ...identity(), "content-type": "application/json", origin: "https://qm.example.test" };
const offer = JSON.stringify({ sdp: "v=0\r\no=offer" });
test.after(() => {
- core.close();
surface.close();
});
@@ -90,13 +66,12 @@ test("voice route authenticates and gates accounts before spending", async () =>
403,
);
}
- assert.equal(claims, 0);
const capability = await fetch(`${base}/api/voice`, { headers: identity() });
assert.deepEqual(await capability.json(), { enabled: true, model: "gpt-live-1" });
assert.equal(capability.headers.get("cache-control"), "no-store");
});
-test("voice rejects invalid and oversized offers before claiming a slot", async () => {
+test("voice rejects invalid and oversized offers", async () => {
for (const body of ["{", "null", "[]", "{}", JSON.stringify({ sdp: "bad" })]) {
assert.equal((await fetch(`${base}/api/voice/sessions`, { method: "POST", headers, body })).status, 400);
}
@@ -110,16 +85,6 @@ test("voice rejects invalid and oversized offers before claiming a slot", async
).status,
413,
);
- assert.equal(claims, 0);
-});
-
-test("voice fails closed on rate limits and durable-store outages", async () => {
- claimAvailable = false;
- assert.equal((await fetch(`${base}/api/voice/sessions`, { method: "POST", headers, body: offer })).status, 429);
- claimStatus = 503;
- assert.equal((await fetch(`${base}/api/voice/sessions`, { method: "POST", headers, body: offer })).status, 503);
- claimStatus = 200;
- claimAvailable = true;
});
test("voice relays only a sanitized SDP result and never a provider key", async (t) => {
diff --git a/plugins/web-ui/test/speech-to-text-route.test.ts b/plugins/web-ui/test/speech-to-text-route.test.ts
index 02037e1d1..6648a6a69 100644
--- a/plugins/web-ui/test/speech-to-text-route.test.ts
+++ b/plugins/web-ui/test/speech-to-text-route.test.ts
@@ -1,37 +1,23 @@
-import { test } from "node:test";
+import { test, type TestContext } from "node:test";
import assert from "node:assert/strict";
-import { createServer } from "node:http";
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import { mkdir, mkdtemp, readdir, rm, stat, utimes, writeFile } from "node:fs/promises";
+import { Writable } from "node:stream";
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";
-let claimStatus = 200;
-let claimAvailable = true;
-let claims = 0;
-const core = createServer((req, res) => {
- if (!req.url?.startsWith("/v1/auth/broker/claim")) {
- res.writeHead(200, { "content-type": "application/json" });
- res.end("{}");
- return;
- }
- let body = "";
- req.on("data", (chunk) => {
- body += chunk;
- });
- req.on("end", () => {
- claims++;
- const input = JSON.parse(body);
- assert.equal(input.ids.length, 60);
- res.writeHead(claimStatus, { "content-type": "application/json" });
- res.end(JSON.stringify({ claimed: claimAvailable ? input.ids[0] : null }));
- });
-});
-await new Promise<void>((resolve) => core.listen(0, resolve));
-process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;
+const testTmp = await mkdtemp(join(tmpdir(), "dictation-route-test-"));
+process.env.TMPDIR = testTmp;
process.env.CORE_SIGNING_SECRET = "voice-route-test";
process.env.WEB_UI_PUBLIC_URL = "https://qm.example.test";
process.env.WEB_UI_VOICE_USERS = "alice";
process.env.OPENAI_API_KEY = "voice-test-key";
const { handler } = await import("../server/index.ts");
+const { sweepDictationSpool } = await import("../server/voice-routes.ts");
const surface = createServer((req, res) => void handler(req, res));
await new Promise<void>((resolve) => surface.listen(0, resolve));
const base = `http://localhost:${(surface.address() as AddressInfo).port}`;
@@ -40,9 +26,33 @@ const identity = (user = "alice") => ({
});
const headers = { ...identity(), "content-type": "audio/mp4", origin: "https://qm.example.test" };
+let openai: (req: IncomingMessage, res: ServerResponse) => void = (_req, res) => res.end("{}");
+const openaiStub = createServer((req, res) => openai(req, res));
+await new Promise<void>((resolve) => openaiStub.listen(0, resolve));
+const spoolDir = join(tmpdir(), "qm-dictation");
+const spooled = async () => new Set(await readdir(spoolDir).catch(() => []));
+
+function routeOpenAIToStub(t: TestContext) {
+ const actualFetch = globalThis.fetch;
+ t.mock.method(globalThis, "fetch", (input: Parameters<typeof fetch>[0], init?: RequestInit) =>
+ actualFetch(
+ String(input).replace("https://api.openai.com", `http://localhost:${(openaiStub.address() as AddressInfo).port}`),
+ init,
+ ),
+ );
+}
+
+function reply(status: number, body: unknown, headers: Record<string, string> = {}) {
+ return (_req: IncomingMessage, res: ServerResponse) => {
+ res.writeHead(status, { "content-type": "application/json", ...headers });
+ res.end(JSON.stringify(body));
+ };
+}
+
test.after(() => {
- core.close();
surface.close();
+ openaiStub.close();
+ void rm(testTmp, { recursive: true, force: true });
});
test("dictation checks identity, origin, allowlist and format before provider access", async () => {
@@ -68,19 +78,158 @@ test("dictation checks identity, origin, allowlist and format before provider ac
expected,
);
}
- assert.equal(claims, 0);
const capability = await fetch(`${base}/api/dictation`, { headers: identity() });
assert.deepEqual(await capability.json(), { enabled: true, provider: "OpenAI" });
assert.equal(capability.headers.get("cache-control"), "no-store");
});
-test("dictation uses durable limits and fails closed on unavailable rate storage", async () => {
- claimAvailable = false;
- assert.equal((await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "audio" })).status, 429);
- claimAvailable = true;
- claimStatus = 503;
- assert.equal((await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "audio" })).status, 503);
- claimStatus = 200;
+test("dictation uploads the whole recording to OpenAI and removes its spool file", async (t) => {
+ routeOpenAIToStub(t);
+ const before = await spooled();
+ const audio = new Uint8Array(30 * 1024 * 1024).map((_, i) => (i * 31) & 255);
+ let received = "";
+ openai = (req, res) => {
+ const hash = createHash("sha256");
+ const chunks: Buffer[] = [];
+ req.on("data", (chunk: Buffer) => chunks.push(chunk));
+ req.on("end", () => {
+ void new Response(Buffer.concat(chunks), { headers: { "content-type": req.headers["content-type"]! } })
+ .formData()
+ .then(async (form) => {
+ const file = form.get("file") as File;
+ received = hash.update(Buffer.from(await file.arrayBuffer())).digest("hex");
+ assert.equal(form.get("model"), "gpt-transcribe");
+ assert.equal(file.type, "audio/mp4");
+ reply(200, { text: " long recording " })(req, res);
+ });
+ });
+ };
+ const response = await fetch(`${base}/api/dictation`, { method: "POST", headers, body: audio });
+ assert.deepEqual(await response.json(), { text: "long recording" });
+ assert.equal(received, createHash("sha256").update(audio).digest("hex"));
+ assert.deepEqual(await spooled(), before);
+});
+
+test("dictation survives OpenAI answering before reading the upload", async (t) => {
+ routeOpenAIToStub(t);
+ openai = reply(429, { error: { code: "rate_limit_exceeded" } }, { "retry-after": "7" });
+ const response = await fetch(`${base}/api/dictation`, {
+ method: "POST",
+ headers,
+ body: new Uint8Array(5 * 1024 * 1024),
+ });
+ assert.equal(response.status, 429);
+ assert.equal(response.headers.get("retry-after"), "7");
+ assert.equal((await fetch(`${base}/api/dictation`, { headers: identity() })).status, 200);
+});
+
+test("dictation maps OpenAI errors by code", async (t) => {
+ routeOpenAIToStub(t);
+ const post = async () => {
+ const response = await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "audio" });
+ return { status: response.status, retryAfter: response.headers.get("retry-after"), ...(await response.json()) };
+ };
+ const cases: [number, string, Record<string, string>, Record<string, unknown>][] = [
+ [
+ 429,
+ "rate_limit_exceeded",
+ { "retry-after": "12" },
+ { status: 429, retryAfter: "12", error: "provider_rate_limit" },
+ ],
+ [429, "rate_limit_exceeded", {}, { retryAfter: null, error: "provider_rate_limit" }],
+ [429, "rate_limit_exceeded", { "retry-after": "" }, { retryAfter: null }],
+ [429, "rate_limit_exceeded", { "retry-after": "1e3" }, { retryAfter: null }],
+ [429, "rate_limit_exceeded", { "retry-after": "-5" }, { retryAfter: null }],
+ [429, "rate_limit_exceeded", { "retry-after": "Wed, 21 Oct 2015 07:28:00 GMT" }, { retryAfter: null }],
+ [429, "rate_limit_exceeded", { "retry-after": "99999" }, { retryAfter: "3600" }],
+ [429, "insufficient_quota", {}, { status: 503, error: "dictation_unavailable" }],
+ [413, "file_too_large", {}, { status: 413, error: "audio_too_large" }],
+ [400, "unsupported_format", {}, { status: 400, error: "invalid_audio" }],
+ [400, "invalid_value", {}, { status: 502, error: "dictation_rejected" }],
+ [400, "invalid_parameter", {}, { status: 502, error: "dictation_rejected" }],
+ [401, "invalid_api_key", {}, { status: 502, error: "dictation_rejected" }],
+ [409, "conflict", {}, { status: 503, error: "dictation_unavailable" }],
+ [302, "redirect", { location: "https://elsewhere.test/" }, { status: 503, error: "dictation_unavailable" }],
+ [500, "server_error", {}, { status: 503, error: "dictation_unavailable" }],
+ ];
+ for (const [status, code, providerHeaders, expected] of cases) {
+ openai = reply(status, { error: { code, message: "x".repeat(1000) } }, providerHeaders);
+ const result = await post();
+ for (const [key, value] of Object.entries(expected))
+ assert.equal(result[key], value, `${status} ${code} ${JSON.stringify(providerHeaders)}: ${key}`);
+ }
+ openai = reply(400, { error: { code: "invalid_value", param: "file" } });
+ assert.equal((await post()).error, "invalid_audio");
+});
+
+test("dictation reports storage failures as unavailable instead of hanging", async (t) => {
+ routeOpenAIToStub(t);
+ const actual = fs.createWriteStream;
+ t.mock.method(fs, "createWriteStream", (path: string, options: object) => {
+ actual(path, options).destroy();
+ return new Writable({
+ write(_chunk, _encoding, done) {
+ done(Object.assign(new Error("no space left on device"), { code: "ENOSPC", syscall: "write" }));
+ },
+ });
+ });
+ const response = await fetch(`${base}/api/dictation`, {
+ method: "POST",
+ headers,
+ body: new Uint8Array(256 * 1024),
+ signal: AbortSignal.timeout(5_000),
+ });
+ assert.equal(response.status, 503);
+ assert.equal((await response.json()).error, "dictation_unavailable");
+ assert.deepEqual(await spooled(), new Set());
+});
+
+test("startup sweep removes only stale spool files", async () => {
+ await mkdir(spoolDir, { recursive: true });
+ const stale = join(spoolDir, "stale-test");
+ const fresh = join(spoolDir, "fresh-test");
+ await writeFile(stale, "old");
+ await writeFile(fresh, "new");
+ const old = new Date(Date.now() - 2 * 3_600_000);
+ await utimes(stale, old, old);
+ await sweepDictationSpool();
+ await assert.rejects(stat(stale));
+ assert.ok(await stat(fresh));
+ await sweepDictationSpool(Date.now() + 2 * 3_600_000);
+ assert.deepEqual(await spooled(), new Set());
+});
+
+test("empty and interrupted uploads never reach OpenAI or leave spool files", async (t) => {
+ routeOpenAIToStub(t);
+ const before = await spooled();
+ let calls = 0;
+ openai = (req, res) => {
+ calls++;
+ reply(200, { text: "unexpected" })(req, res);
+ };
+ assert.equal(
+ (await (await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "" })).json()).error,
+ "empty_audio",
+ );
+ const controller = new AbortController();
+ const body = new ReadableStream<Uint8Array>({
+ start(stream) {
+ stream.enqueue(new Uint8Array(64 * 1024));
+ setTimeout(() => controller.abort(), 100);
+ },
+ });
+ await assert.rejects(
+ fetch(`${base}/api/dictation`, {
+ method: "POST",
+ headers,
+ body,
+ duplex: "half",
+ signal: controller.signal,
+ } as RequestInit),
+ );
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ assert.equal(calls, 0);
+ assert.deepEqual(await spooled(), before);
});
test("dictation returns only transcript text and sanitizes provider failures", async (t) => {
@@ -103,9 +252,8 @@ test("dictation returns only transcript text and sanitizes provider failures", a
assert.equal(native.status, 200);
failure = true;
const rejected = await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "audio" });
- assert.equal(rejected.status, 503);
+ assert.equal(rejected.status, 502);
assert.equal((await rejected.text()).includes("secret details"), false);
- assert.equal((await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "" })).status, 400);
});
test("disconnecting a dictation request cancels its provider request", async (t) => {
@@ -134,24 +282,6 @@ test("disconnecting a dictation request cancels its provider request", async (t)
await Promise.all([rejected, cancelled.promise]);
});
-test("chunked recordings cannot exceed the audio limit", async () => {
- const chunk = new Uint8Array(1024 * 1024);
- let sent = 0;
- const body = new ReadableStream<Uint8Array>({
- pull(controller) {
- if (sent++ > 10) controller.close();
- else controller.enqueue(chunk);
- },
- });
- const response = await fetch(`${base}/api/dictation`, {
- method: "POST",
- headers,
- body,
- duplex: "half",
- } as RequestInit);
- assert.equal(response.status, 413);
-});
-
test("provider network failures log their cause chain", async (t) => {
const actualFetch = globalThis.fetch;
t.mock.method(globalThis, "fetch", async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
@@ -162,7 +292,9 @@ test("provider network failures log their cause chain", async (t) => {
t.mock.method(console, "error", (line: unknown) => void logged.push(String(line)));
const result = await fetch(`${base}/api/dictation`, { method: "POST", headers, body: "audio" });
assert.equal(result.status, 503);
- const line = logged.find((entry) => entry.startsWith("[voice] dictation_provider:"));
+ const line = logged.find(
+ (entry) => entry.startsWith("[voice] dictation_provider:") && entry.includes('"aborted":false'),
+ );
assert.match(line ?? "", /fetch failed <- .*ECONNRESET.*socket hang up/);
assert.match(line ?? "", /"path":"audio\/transcriptions"/);
});
diff --git a/plugins/web-ui/test/voice.test.ts b/plugins/web-ui/test/voice.test.ts
index 55bd35d77..86993f472 100644
--- a/plugins/web-ui/test/voice.test.ts
+++ b/plugins/web-ui/test/voice.test.ts
@@ -1,4 +1,7 @@
import { test, type TestContext } from "node:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import assert from "node:assert/strict";
import type { IncomingMessage } from "node:http";
import { createLiveVoice, transcribe, voiceEnabled, voiceRequestAllowed, VoiceProviderError } from "../server/voice.ts";
@@ -11,6 +14,15 @@ const caller = {
};
const stubFetch = (t: TestContext, impl: (url: string, init?: RequestInit) => Promise<Response>) =>
t.mock.method(globalThis, "fetch", (url: Parameters<typeof fetch>[0], init?: RequestInit) => impl(String(url), init));
+const audioDirs: string[] = [];
+test.after(() => Promise.all(audioDirs.map((dir) => rm(dir, { recursive: true, force: true }))));
+const audioFile = async (bytes: number[]) => {
+ const dir = await mkdtemp(join(tmpdir(), "voice-test-"));
+ audioDirs.push(dir);
+ const file = join(dir, "audio");
+ await writeFile(file, new Uint8Array(bytes));
+ return file;
+};
const req = (headers: Record<string, string>) => ({ headers }) as unknown as IncomingMessage;
test("voice requires explicit rollout, key, signing, and account access", () => {
@@ -34,7 +46,7 @@ test("voice requests come from the web origin or a verified native client", () =
test("session creation keeps credentials and authoritative configuration on server", async (t) => {
stubFetch(t, async (url, init) => {
assert.equal(url, "https://api.openai.com/v1/live/sessions");
- assert.equal(init?.redirect, "error");
+ assert.equal(init?.redirect, "manual");
const headers = new Headers(init?.headers);
assert.equal(headers.get("authorization"), "Bearer server-only-key");
assert.match(headers.get("OpenAI-Safety-Identifier")!, /^[a-f0-9]{64}$/);
@@ -82,7 +94,7 @@ test("transcription sends audio with the server-selected model and returns only
stubFetch(t, async (url, init) => {
if (responses.length) return responses.shift()!;
assert.equal(url, "https://api.openai.com/v1/audio/transcriptions");
- const form = init!.body as FormData;
+ const form = await new Response(init!.body, { headers: init!.headers }).formData();
assert.equal(form.get("model"), "gpt-transcribe");
const file = form.get("file") as File;
assert.equal(file.name, "dictation.m4a");
@@ -90,7 +102,7 @@ test("transcription sends audio with the server-selected model and returns only
assert.equal(file.size, 3);
return Response.json({ text: " Test dictation. ", internal: "ignored" });
});
- assert.equal(await transcribe(new Uint8Array([1, 2, 3]), "audio/mp4", caller), "Test dictation.");
- responses.push(Response.json({ text: 42 }), Response.json({ text: "a".repeat(100_001) }));
- for (let i = 0; i < 2; i++) await assert.rejects(transcribe(new Uint8Array([1]), "audio/wav", caller));
+ assert.equal(await transcribe(await audioFile([1, 2, 3]), 3, "audio/mp4", caller), "Test dictation.");
+ responses.push(Response.json({ text: 42 }));
+ await assert.rejects(transcribe(await audioFile([1]), 1, "audio/wav", caller), /Invalid transcription/);
});