Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

8.3 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { type VoiceConfig, voiceConfig } from "../src/voice/config";
import { createVoiceProvider } from "../src/voice/provider";

const config: VoiceConfig = {
  provider: "openai-realtime",
  model: "voice-model",
  apiKey: "voice-secret",
  voice: "marin",
};
const offer = "v=0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n";
const input = {
  sdp: offer,
  agentName: "Research assistant",
  agentTitle: "Researcher",
  agentRole: "Explain research clearly.",
  userId: "user-1",
  signal: new AbortController().signal,
};

describe("independent live voice configuration", () => {
  test("chat and dictation keys do not enable live voice", () => {
    expect(
      voiceConfig({
        OPENAI_API_KEY: "chat-secret",
        TRANSCRIPTION_API_KEY: "dictation-secret",
      }),
    ).toBeUndefined();
    expect(() =>
      voiceConfig({
        VOICE_PROVIDER: "openai-realtime",
        VOICE_MODEL: "model",
        OPENAI_API_KEY: "chat-secret",
      }),
    ).toThrow("VOICE_API_KEY");
  });
  test("requires complete configuration and defaults the voice only", () => {
    expect(() => voiceConfig({ VOICE_NAME: "marin" })).toThrow(
      "VOICE_PROVIDER",
    );
    expect(() =>
      voiceConfig({ VOICE_PROVIDER: "other", VOICE_API_KEY: "secret" }),
    ).toThrow("VOICE_PROVIDER");
    expect(() =>
      voiceConfig({
        VOICE_PROVIDER: "openai-realtime",
        VOICE_API_KEY: "secret",
      }),
    ).toThrow("VOICE_MODEL");
    expect(
      voiceConfig({
        VOICE_PROVIDER: "openai-realtime",
        VOICE_MODEL: "voice-model",
        VOICE_API_KEY: "voice-secret",
      }),
    ).toEqual(config);
    expect(
      voiceConfig({
        VOICE_PROVIDER: "openai-realtime",
        VOICE_MODEL: "voice-model",
        VOICE_API_KEY: "voice-secret",
        VOICE_NAME: "cedar",
      })?.voice,
    ).toBe("cedar");
  });
});

test("creates an audio session with agent delegation and interruption using only the voice key", async () => {
  const provider = createVoiceProvider(config, async (url, init) => {
    expect(url).toBe("https://api.openai.com/v1/realtime/calls");
    expect(init.method).toBe("POST");
    expect(init.redirect).toBe("error");
    expect(init.signal).toBe(input.signal);
    expect(new Headers(init.headers).get("authorization")).toBe(
      "Bearer voice-secret",
    );
    if (!(init.body instanceof FormData)) throw new Error("Expected multipart");
    expect([...init.body.keys()].sort()).toEqual(["sdp", "session"]);
    expect(init.body.get("sdp")).toBe(offer);
    const session = init.body.get("session");
    if (typeof session !== "string") throw new Error("Expected session JSON");
    expect(session).not.toContain("voice-secret");
    const parsed = JSON.parse(session);
    expect(parsed).toMatchObject({
      type: "realtime",
      model: "voice-model",
      output_modalities: ["audio"],
      audio: {
        input: {
          transcription: { model: "gpt-4o-mini-transcribe" },
          turn_detection: {
            type: "semantic_vad",
            create_response: true,
            interrupt_response: true,
          },
        },
        output: { voice: "marin" },
      },
      tools: [
        {
          type: "function",
          name: "ask_agent",
          parameters: {
            type: "object",
            properties: { request: { type: "string", maxLength: 4000 } },
            required: ["request"],
            additionalProperties: false,
          },
        },
      ],
    });
    expect(parsed.instructions).toContain("Research assistant");
    expect(parsed.instructions).toContain("Explain research clearly.");
    expect(parsed.instructions).toContain("brainstorm directly");
    expect(parsed.instructions).not.toContain("Delegate every substantive");
    expect(parsed.instructions).toContain("CRITICAL:");
    expect(parsed.instructions).toContain("ask_agent");
    return new Response(offer, {
      headers: { "content-type": "application/sdp" },
    });
  });
  expect(await provider.connect(input)).toEqual({
    transport: "webrtc",
    sdp: offer,
  });
});

test("provider failures and malformed successes are safe and bounded", async () => {
  for (const response of [
    new Response("voice-secret", { status: 401 }),
    new Response("voice-secret", { status: 429 }),
    new Response('{"key":"voice-secret"}'),
    new Response(`${offer}${"x".repeat(128 * 1024)}`),
  ]) {
    const provider = createVoiceProvider(config, async () => response);
    try {
      await provider.connect(input);
      throw new Error("Expected rejection");
    } catch (error) {
      expect(error).toBeInstanceOf(Error);
      expect(String(error)).not.toContain("voice-secret");
      expect(String(error)).toContain("voice service");
    }
  }
  const provider = createVoiceProvider(config, async () => {
    throw new Error("voice-secret");
  });
  await expect(provider.connect(input)).rejects.toThrow("voice service");
});

test("aborted requests never start an upstream call", async () => {
  const controller = new AbortController();
  controller.abort();
  let calls = 0;
  const provider = createVoiceProvider(config, async () => {
    calls++;
    return new Response(offer);
  });
  await expect(
    provider.connect({ ...input, signal: controller.signal }),
  ).rejects.toThrow();
  expect(calls).toBe(0);
});

test("Grok uses an independent voice default and only returns an ephemeral browser credential", async () => {
  const xaiConfig = voiceConfig({
    VOICE_PROVIDER: "xai-realtime",
    VOICE_MODEL: "grok-voice-latest",
    VOICE_API_KEY: "xai-server-secret",
  });
  expect(xaiConfig?.voice).toBe("ara");
  if (!xaiConfig) throw new Error("Expected config");
  const provider = createVoiceProvider(xaiConfig, async (url, init) => {
    expect(url).toBe("https://api.x.ai/v1/realtime/client_secrets");
    expect(new Headers(init.headers).get("authorization")).toBe(
      "Bearer xai-server-secret",
    );
    expect(init.redirect).toBe("error");
    expect(init.signal).toBe(input.signal);
    expect(init.body).toBe(JSON.stringify({ expires_after: { seconds: 300 } }));
    return Response.json({
      value: "ephemeral-token",
      expires_at: Math.floor(Date.now() / 1000) + 300,
      extra: "xai-server-secret",
    });
  });
  expect(provider.transport).toBe("websocket");
  const connection = await provider.connect({ ...input, sdp: undefined });
  expect(connection).toMatchObject({
    transport: "websocket",
    clientSecret: "ephemeral-token",
    url: "wss://api.x.ai/v1/realtime?model=grok-voice-latest",
    session: {
      voice: "ara",
      turn_detection: { type: "server_vad" },
      audio: {
        input: { format: { type: "audio/pcm", rate: 24000 } },
        output: { format: { type: "audio/pcm", rate: 24000 } },
      },
      tools: [{ type: "function", name: "ask_agent" }],
    },
  });
  expect(JSON.stringify(connection)).not.toContain("xai-server-secret");
});

test("Grok rejects malformed, expired, or reflected permanent credentials", async () => {
  const expires_at = Math.floor(Date.now() / 1000) + 300;
  for (const body of [
    {},
    { value: "token" },
    { value: "token", expires_at: 0 },
    { value: "voice-secret", expires_at },
    { value: "token with spaces", expires_at },
  ]) {
    const provider = createVoiceProvider(
      { ...config, provider: "xai-realtime" },
      async () => Response.json(body),
    );
    await expect(provider.connect(input)).rejects.toThrow("voice service");
  }
});

test("Learning-enabled voice sends substantive work through the selected channel agent", async () => {
  const provider = createVoiceProvider(config, async (_url, init) => {
    if (!(init.body instanceof FormData)) throw new Error("Expected multipart");
    const value = init.body.get("session");
    if (typeof value !== "string") throw new Error("Expected session JSON");
    const session = JSON.parse(value);
    expect(session.instructions).toContain("Delegate every substantive");
    expect(session.instructions).toContain("published Skills");
    expect(session.instructions).not.toContain("brainstorm directly");
    expect(session.instructions).toContain("acknowledgments");
    expect(session.tools.map((tool: { name: string }) => tool.name)).toEqual([
      "ask_agent",
    ]);
    return new Response(offer, {
      headers: { "content-type": "application/sdp" },
    });
  });
  await provider.connect({ ...input, learningEnabled: true });
});