Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

11.4 KB
Raw Blame
import { expect, test } from "bun:test";
import type { Message } from "@ag-ui/core";
import { askChannelAgent, voiceContext } from "../src/lib/voice/agent-bridge";
import { decodePcm, encodePcm } from "../src/lib/voice/pcm";
import { VoiceSession } from "../src/lib/voice/session";
import type {
  ConnectVoice,
  VoiceConnection,
  VoiceEvent,
} from "../src/lib/voice/types";

function setup(
  askAgent: (
    request: string,
    signal: AbortSignal,
  ) => Promise<string> = async () => "Agent result",
  context: () => string = () => "user: Old request",
) {
  let receive: (event: VoiceEvent) => void = () => {};
  let closed = 0;
  let muted = false;
  const sent: VoiceEvent[] = [];
  const connection: VoiceConnection = {
    stream: new EventTarget() as MediaStream,
    send: (event) => {
      sent.push(event);
    },
    mute: (value) => {
      muted = value;
    },
    close: () => {
      closed++;
    },
  };
  const connect: ConnectVoice = async (options) => {
    receive = options.onEvent;
    return connection;
  };
  const session = new VoiceSession({
    channelId: "channel-1",
    connect,
    askAgent,
    context,
  });
  return {
    session,
    sent,
    emit: (event: VoiceEvent) => receive(event),
    closed: () => closed,
    muted: () => muted,
  };
}
const tool = (id = "call-1", request = "Find a meeting time") => ({
  type: "response.function_call_arguments.done",
  name: "ask_agent",
  call_id: id,
  arguments: JSON.stringify({ request }),
});
const tick = async () => {
  await Promise.resolve();
  await Promise.resolve();
};

test("joining a call loads thread context without asking the agent to speak", async () => {
  const fixture = setup();
  try {
    await fixture.session.start();
    expect(fixture.session.getSnapshot().phase).toBe("connected");
    expect(JSON.stringify(fixture.sent)).toContain("Old request");
    expect(fixture.sent.some((event) => event.type === "response.create")).toBe(
      false,
    );
    expect(fixture.session.getSnapshot().speaking).toBe(false);
  } finally {
    fixture.session.end();
  }
});

test("muting the caller leaves agent playback and replies active", async () => {
  const fixture = setup();
  try {
    await fixture.session.start();
    fixture.emit({ type: "output_audio_buffer.started" });
    fixture.session.mute();
    fixture.emit({
      type: "response.output_audio_transcript.delta",
      delta: "Still speaking",
    });
    expect(fixture.muted()).toBe(true);
    expect(fixture.session.getSnapshot()).toMatchObject({
      muted: true,
      speaking: true,
      reply: "Still speaking",
    });
    expect(fixture.closed()).toBe(0);
    fixture.session.mute();
    expect(fixture.muted()).toBe(false);
  } finally {
    fixture.session.end();
  }
});

test("voice delegates once even when the provider repeats its function call in response.done", async () => {
  const pending = Promise.withResolvers<string>();
  const requests: string[] = [];
  const fixture = setup(async (text) => {
    requests.push(text);
    return pending.promise;
  });
  try {
    await fixture.session.start();
    fixture.emit(tool());
    fixture.emit({
      type: "response.done",
      response: { output: [{ ...tool(), type: "function_call" }] },
    });
    expect(requests).toEqual(["Find a meeting time"]);
    expect(fixture.session.getSnapshot().working).toBe(true);
    pending.resolve("Tuesday at 10");
    await tick();
    await tick();
    const results = fixture.sent.filter(
      (event) =>
        event.type === "conversation.item.create" &&
        JSON.stringify(event.item).includes("function_call_output"),
    );
    expect(results).toHaveLength(1);
    expect(JSON.stringify(results)).toContain("Tuesday at 10");
    expect(fixture.session.getSnapshot().working).toBe(false);
  } finally {
    fixture.session.end();
  }
});

test("hangup ignores late tool results and closes the transport", async () => {
  const pending = Promise.withResolvers<string>();
  const fixture = setup(async () => pending.promise);
  await fixture.session.start();
  fixture.emit(tool());
  fixture.session.end();
  const count = fixture.sent.length;
  pending.resolve("Late result");
  await tick();
  await tick();
  expect(fixture.sent).toHaveLength(count);
  expect(fixture.closed()).toBe(1);
  expect(fixture.session.getSnapshot().phase).toBe("idle");
});

test("hangup during connection closes a late transport and never sends a greeting", async () => {
  const pending = Promise.withResolvers<VoiceConnection>();
  let closed = false;
  const session = new VoiceSession({
    channelId: "channel-1",
    connect: async () => pending.promise,
    context: () => "",
    askAgent: async () => "",
  });
  const start = session.start();
  await tick();
  session.end();
  pending.resolve({
    stream: new EventTarget() as MediaStream,
    close: () => {
      closed = true;
    },
    mute() {},
    send() {
      throw new Error("Must not send after hangup");
    },
  });
  await start;
  expect(closed).toBe(true);
  expect(session.getSnapshot().phase).toBe("idle");
});

test("an overlapping voice tool request is refused without executing a second task", async () => {
  const pending = Promise.withResolvers<string>();
  let count = 0;
  const fixture = setup(async () => {
    count++;
    return pending.promise;
  });
  try {
    await fixture.session.start();
    fixture.emit(tool());
    fixture.emit(tool("call-2", "Delete it instead"));
    await tick();
    expect(count).toBe(1);
    expect(JSON.stringify(fixture.sent)).toContain("still working");
    expect(fixture.session.getSnapshot().working).toBe(true);
    fixture.emit({ type: "response.done", response: { output: [] } });
    expect(
      fixture.sent.filter((event) => event.type === "response.create"),
    ).toHaveLength(0);
  } finally {
    fixture.session.end();
    pending.resolve("Finished");
  }
});

test("a VAD response consumes tool results received during user speech without a duplicate reply", async () => {
  const pending = Promise.withResolvers<string>();
  const fixture = setup(async () => pending.promise);
  try {
    await fixture.session.start();
    fixture.emit({ type: "response.done", response: { output: [] } });
    fixture.emit(tool());
    fixture.emit({ type: "input_audio_buffer.speech_started" });
    pending.resolve("Current result");
    await tick();
    await tick();
    fixture.emit({ type: "input_audio_buffer.speech_stopped" });
    fixture.emit({ type: "response.created" });
    fixture.emit({ type: "response.done", response: { output: [] } });
    expect(
      fixture.sent.filter((event) => event.type === "response.create"),
    ).toHaveLength(0);
  } finally {
    fixture.session.end();
  }
});

test("provider and agent failures are surfaced; mute updates microphone ownership", async () => {
  const fixture = setup(async () => {
    throw new Error("The agent is unavailable");
  });
  await fixture.session.start();
  fixture.session.mute();
  expect(fixture.muted()).toBe(true);
  fixture.emit(tool());
  await tick();
  expect(JSON.stringify(fixture.sent)).toContain("agent is unavailable");
  fixture.emit({ type: "error", error: { code: "invalid_request" } });
  expect(fixture.session.getSnapshot().phase).toBe("error");
  expect(fixture.closed()).toBe(1);
  fixture.session.end();
});

test("bad tool arguments cannot run agent work", async () => {
  let count = 0;
  const fixture = setup(async () => {
    count++;
    return "";
  });
  try {
    await fixture.session.start();
    fixture.emit({ ...tool(), arguments: "{}" });
    fixture.emit({ ...tool("call-2"), name: "delete_everything" });
    await tick();
    expect(count).toBe(0);
    expect(JSON.stringify(fixture.sent)).toContain("error");
  } finally {
    fixture.session.end();
  }
});

test("voice bridge returns only the new agent answer and respects shared thread busy state", async () => {
  const messages: Message[] = [
    { id: "old", role: "assistant", content: "Old answer" },
  ];
  let busy = true;
  const requests: string[] = [];
  const channel = {
    busy: () => busy,
    send: async (request: string) => {
      requests.push(request);
      messages.push({ id: "new", role: "assistant", content: "New answer" });
      return "New answer";
    },
  };
  const signal = new AbortController().signal;
  await expect(askChannelAgent("Hello", signal, channel)).rejects.toThrow(
    "still working",
  );
  expect(requests).toEqual([]);
  busy = false;
  expect(await askChannelAgent("Hello", signal, channel)).toBe("New answer");
  expect(requests).toEqual(["Hello"]);
});

// An emoji is two UTF-16 units; each text below is one unit over its limit, so a plain slice
// would keep half of the emoji at the cut.
const emoji = "\u{1F600}";

test("the call's opening context is cut between characters", async () => {
  const fixture = setup(undefined, () => `${emoji.repeat(6000)}.`);
  try {
    await fixture.session.start();
    const item = fixture.sent.find(
      (event) => event.type === "conversation.item.create",
    )?.item as { content: { text: string }[] } | undefined;
    const text = item?.content[0]?.text ?? "";
    expect(text.isWellFormed()).toBe(true);
    expect(text.endsWith(`${emoji.repeat(5999)}.`)).toBe(true);
  } finally {
    fixture.session.end();
  }
});

test("live captions are cut between characters", async () => {
  const fixture = setup();
  try {
    await fixture.session.start();
    fixture.emit({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "user-1",
      transcript: `.${emoji.repeat(2000)}`,
    });
    expect(fixture.session.getSnapshot().transcript.isWellFormed()).toBe(true);
    fixture.emit({ type: "response.created" });
    fixture.emit({
      type: "response.output_audio_transcript.delta",
      item_id: "assistant-1",
      delta: `${emoji.repeat(2000)}.`,
    });
    expect(fixture.session.getSnapshot().reply.isWellFormed()).toBe(true);
    expect(fixture.session.getSnapshot().reply).toBe(`${emoji.repeat(1999)}.`);
  } finally {
    fixture.session.end();
  }
});

test("a long delegated answer is cut between characters", async () => {
  const fixture = setup(async () => `.${emoji.repeat(8000)}`);
  try {
    await fixture.session.start();
    fixture.emit(tool());
    await tick();
    await tick();
    const output = fixture.sent
      .map((event) => event.item as { type?: string; output?: string })
      .find((item) => item?.type === "function_call_output")?.output;
    const answer = (JSON.parse(output ?? "{}") as { answer?: string }).answer;
    expect(answer?.isWellFormed()).toBe(true);
    expect(answer).toBe(`.${emoji.repeat(7999)}`);
  } finally {
    fixture.session.end();
  }
});

test("chat context for a call is cut between characters", () => {
  const context = voiceContext([
    { id: "user", role: "user", content: `${emoji.repeat(6000)}.` },
  ]);
  expect(context.isWellFormed()).toBe(true);
  expect(context).toBe(`${emoji.repeat(5999)}.`);
});

test("voice context excludes system instructions and tool payloads", () => {
  expect(
    voiceContext([
      { id: "system", role: "system", content: "Private instructions" },
      { id: "user", role: "user", content: "Hello" },
      {
        id: "tool",
        role: "tool",
        toolCallId: "call",
        content: "Raw tool payload",
      },
    ]),
  ).toBe("user: Hello");
});

test("Grok PCM transport preserves signed little-endian audio samples", () => {
  const bytes = new Uint8Array([0, 128, 0, 0, 255, 127]);
  expect([...decodePcm(encodePcm(bytes.buffer))]).toEqual([
    -1,
    0,
    32767 / 32768,
  ]);
  expect(() => decodePcm(btoa("x"))).toThrow("Invalid audio");
});