Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(frontend/copilot): Configurable transcription endpoints (#13500)

zhifu gao47327d7bbc7c
diff --git a/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts b/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts
new file mode 100644
index 0000000000..e553c735c7
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts
@@ -0,0 +1,155 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { NextRequest } from "next/server";
+
+vi.mock("@/lib/autogpt-server-api/helpers", () => ({
+ getServerAuthToken: vi.fn(),
+}));
+
+import { getServerAuthToken } from "@/lib/autogpt-server-api/helpers";
+import { POST } from "../route";
+
+function makeAudioRequest(type = "audio/webm"): NextRequest {
+ const formData = new FormData();
+ formData.append("audio", new Blob(["audio-bytes"], { type }));
+ return new Request("http://localhost/api/transcribe", {
+ method: "POST",
+ body: formData,
+ }) as NextRequest;
+}
+
+describe("transcribe route", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.unstubAllGlobals();
+ vi.mocked(getServerAuthToken).mockResolvedValue("session-token");
+ delete process.env.OPENAI_API_KEY;
+ delete process.env.OPENAI_API_BASE_URL;
+ delete process.env.TRANSCRIPTION_API_KEY;
+ delete process.env.TRANSCRIPTION_API_BASE_URL;
+ delete process.env.TRANSCRIPTION_MODEL;
+ });
+
+ it("sends audio to a configured OpenAI-compatible transcription endpoint", async () => {
+ process.env.TRANSCRIPTION_API_BASE_URL = "http://funasr.local:8000/v1/";
+ process.env.TRANSCRIPTION_MODEL = "iic/SenseVoiceSmall";
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ text: "hello from funasr" }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const response = await POST(makeAudioRequest());
+
+ expect(response.status).toBe(200);
+ await expect(response.json()).resolves.toEqual({
+ text: "hello from funasr",
+ });
+ expect(fetchMock).toHaveBeenCalledWith(
+ "http://funasr.local:8000/v1/audio/transcriptions",
+ expect.objectContaining({
+ method: "POST",
+ }),
+ );
+ const fetchInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
+ expect(fetchInit.headers).toBeInstanceOf(Headers);
+ expect((fetchInit.headers as Headers).has("Authorization")).toBe(false);
+ const upstreamBody = fetchInit.body as FormData;
+ expect(upstreamBody.get("model")).toBe("iic/SenseVoiceSmall");
+ const file = upstreamBody.get("file") as File;
+ expect(file.name).toBe("recording.webm");
+ });
+
+ it("uses a transcription API key when one is configured separately from the OpenAI key", async () => {
+ process.env.TRANSCRIPTION_API_BASE_URL = "http://funasr.local:8000/v1";
+ process.env.TRANSCRIPTION_API_KEY = "self-hosted-token";
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ text: "ok" }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await POST(makeAudioRequest());
+
+ const fetchInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
+ expect(fetchInit.headers).toBeInstanceOf(Headers);
+ expect((fetchInit.headers as Headers).get("Authorization")).toBe(
+ "Bearer self-hosted-token",
+ );
+ });
+
+ it("uses the OpenAI API key for the default transcription endpoint", async () => {
+ process.env.OPENAI_API_KEY = "openai-token";
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ text: "ok" }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await POST(makeAudioRequest());
+
+ expect(fetchMock).toHaveBeenCalledWith(
+ "https://api.openai.com/v1/audio/transcriptions",
+ expect.objectContaining({
+ headers: expect.any(Headers),
+ }),
+ );
+ const fetchInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
+ expect((fetchInit.headers as Headers).get("Authorization")).toBe(
+ "Bearer openai-token",
+ );
+ });
+
+ it("does not send the OpenAI API key to a custom transcription endpoint", async () => {
+ process.env.TRANSCRIPTION_API_BASE_URL = "http://funasr.local:8000/v1";
+ process.env.OPENAI_API_KEY = "openai-token-for-other-features";
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ text: "ok" }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await POST(makeAudioRequest());
+
+ const fetchInit = fetchMock.mock.calls[0]?.[1] as RequestInit;
+ expect(fetchInit.headers).toBeInstanceOf(Headers);
+ expect((fetchInit.headers as Headers).has("Authorization")).toBe(false);
+ });
+
+ it("returns upstream transcription API error messages", async () => {
+ process.env.OPENAI_API_KEY = "openai-token";
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(
+ JSON.stringify({ error: { message: "model is unavailable" } }),
+ {
+ status: 503,
+ headers: { "content-type": "application/json" },
+ },
+ ),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ const response = await POST(makeAudioRequest());
+
+ expect(response.status).toBe(503);
+ await expect(response.json()).resolves.toEqual({
+ error: "model is unavailable",
+ });
+ });
+
+ it("keeps requiring an API key for the default OpenAI transcription endpoint", async () => {
+ const response = await POST(makeAudioRequest());
+
+ expect(response.status).toBe(401);
+ await expect(response.json()).resolves.toEqual({
+ error: "OpenAI API key not configured",
+ });
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/api/transcribe/route.ts b/autogpt_platform/frontend/src/app/api/transcribe/route.ts
index 8347176704..5604198485 100644
--- a/autogpt_platform/frontend/src/app/api/transcribe/route.ts
+++ b/autogpt_platform/frontend/src/app/api/transcribe/route.ts
@@ -1,9 +1,41 @@
import { getServerAuthToken } from "@/lib/autogpt-server-api/helpers";
import { NextRequest, NextResponse } from "next/server";
-const WHISPER_API_URL = "https://api.openai.com/v1/audio/transcriptions";
+const DEFAULT_TRANSCRIPTION_API_BASE_URL = "https://api.openai.com/v1";
+const DEFAULT_TRANSCRIPTION_MODEL = "whisper-1";
const MAX_FILE_SIZE = 25 * 1024 * 1024; // 25MB - Whisper's limit
+function getTranscriptionApiBaseUrl(): string {
+ return (
+ process.env.TRANSCRIPTION_API_BASE_URL ||
+ process.env.OPENAI_API_BASE_URL ||
+ DEFAULT_TRANSCRIPTION_API_BASE_URL
+ ).replace(/\/+$/, "");
+}
+
+function getTranscriptionApiUrl(): string {
+ return `${getTranscriptionApiBaseUrl()}/audio/transcriptions`;
+}
+
+function getTranscriptionModel(): string {
+ return process.env.TRANSCRIPTION_MODEL || DEFAULT_TRANSCRIPTION_MODEL;
+}
+
+function getTranscriptionApiKey(): string | undefined {
+ if (process.env.TRANSCRIPTION_API_KEY) {
+ return process.env.TRANSCRIPTION_API_KEY;
+ }
+
+ return usesDefaultOpenAIEndpoint() ? process.env.OPENAI_API_KEY : undefined;
+}
+
+function usesDefaultOpenAIEndpoint(): boolean {
+ return (
+ getTranscriptionApiBaseUrl().toLowerCase() ===
+ DEFAULT_TRANSCRIPTION_API_BASE_URL
+ );
+}
+
function getExtensionFromMimeType(mimeType: string): string {
const subtype = mimeType.split("/")[1]?.split(";")[0];
return subtype || "webm";
@@ -16,9 +48,9 @@ export async function POST(request: NextRequest) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
- const apiKey = process.env.OPENAI_API_KEY;
+ const apiKey = getTranscriptionApiKey();
- if (!apiKey) {
+ if (!apiKey && usesDefaultOpenAIEndpoint()) {
return NextResponse.json(
{ error: "OpenAI API key not configured" },
{ status: 401 },
@@ -46,19 +78,21 @@ export async function POST(request: NextRequest) {
const ext = getExtensionFromMimeType(audioFile.type);
const whisperFormData = new FormData();
whisperFormData.append("file", audioFile, `recording.${ext}`);
- whisperFormData.append("model", "whisper-1");
+ whisperFormData.append("model", getTranscriptionModel());
+ const headers = new Headers();
+ if (apiKey) {
+ headers.set("Authorization", `Bearer ${apiKey}`);
+ }
- const response = await fetch(WHISPER_API_URL, {
+ const response = await fetch(getTranscriptionApiUrl(), {
method: "POST",
- headers: {
- Authorization: `Bearer ${apiKey}`,
- },
+ headers,
body: whisperFormData,
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
- console.error("Whisper API error:", errorData);
+ console.error("Transcription API error:", errorData);
return NextResponse.json(
{ error: errorData.error?.message || "Transcription failed" },
{ status: response.status },