Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

46.5 KB
Raw Blame
const test = require("node:test");
const assert = require("node:assert/strict");
const { EventEmitter } = require("node:events");
const WS = require("ws");

const load = () => import("../../src/helpers/openaiRealtimeStreaming.js");

function makeFakeSocket(readyState) {
  const socket = new EventEmitter();
  socket.readyState = readyState;
  socket.sent = [];
  socket.send = (data) => socket.sent.push(data);
  socket.ping = () => {};
  socket.terminate = () => {
    socket.readyState = WS.CLOSED;
    socket.emit("close", 1006, Buffer.from(""));
  };
  socket.close = () => {
    socket.readyState = WS.CLOSED;
  };
  return socket;
}

async function connectPreconfigured(streaming, socket) {
  const connected = streaming.connect({
    apiKey: "key",
    preconfigured: true,
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));
  await connected;
}

const sentEvents = (socket, type) =>
  socket.sent.map((raw) => JSON.parse(raw)).filter((e) => e.type === type);

async function connectByok(streaming, socket, options) {
  const connected = streaming.connect({
    apiKey: "sk-test",
    createSocket: async () => socket,
    ...options,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));
  socket.emit("message", JSON.stringify({ type: "session.updated" }));
  await connected;
}

function captureLogs(t) {
  const debugLogger = require("../../src/helpers/debugLogger");
  const entries = [];
  for (const level of ["debug", "warn", "error"]) {
    const original = debugLogger[level];
    debugLogger[level] = (message, meta) => entries.push({ level, message, meta });
    t.after(() => {
      debugLogger[level] = original;
    });
  }
  return entries;
}

test("sendAudio buffers frames arriving before the socket exists (token-fetch window)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();

  streaming.beginConnecting();
  assert.equal(streaming.ws, null, "socket not created yet, mirrors the token-fetch window");

  const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));

  assert.equal(sent, false);
  assert.equal(streaming.coldStartBuffer.length, 1);
  assert.equal(streaming.coldStartBufferSize, 4);
});

test("sendAudio drops frames when no connection attempt is in flight (idle/dead instance)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();

  const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));

  assert.equal(sent, false);
  assert.equal(
    streaming.coldStartBuffer.length,
    0,
    "must not buffer forever with no connect in flight"
  );
});

test("sendAudio stops buffering once COLD_START_BUFFER_MAX is reached", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.beginConnecting();

  const chunk = Buffer.alloc(50000, 1);
  streaming.sendAudio(chunk); // size 0 -> 50000
  streaming.sendAudio(chunk); // size 50000 -> 100000
  streaming.sendAudio(chunk); // size 100000 -> 150000 (still under cap when checked)
  streaming.sendAudio(chunk); // size 150000, over the 144000 cap: dropped

  assert.equal(
    streaming.coldStartBuffer.length,
    3,
    "4th chunk must be dropped once the cap is exceeded"
  );
  assert.equal(streaming.coldStartBufferSize, 150000);
});

test("sendAudio flushes buffered audio in order once the socket opens, then sends the live chunk", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.beginConnecting();

  streaming.sendAudio(Buffer.from("first"));
  streaming.sendAudio(Buffer.from("second"));

  streaming.ws = makeFakeSocket(WS.OPEN);
  const sent = streaming.sendAudio(Buffer.from("third"));

  assert.equal(sent, true);
  assert.equal(streaming.ws.sent.length, 3);
  const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);
  assert.deepEqual(payloads, [
    Buffer.from("first").toString("base64"),
    Buffer.from("second").toString("base64"),
    Buffer.from("third").toString("base64"),
  ]);
  assert.equal(streaming.coldStartBuffer.length, 0, "buffer must be cleared after flush");
});

test("connect() preserves audio buffered during beginConnecting() instead of wiping it", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();

  streaming.beginConnecting();
  streaming.sendAudio(Buffer.from("pre-token-fetch audio"));
  assert.equal(streaming.coldStartBuffer.length, 1);

  const socket = makeFakeSocket(WS.CONNECTING);
  const connected = streaming.connect({
    apiKey: "key",
    preconfigured: true,
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(streaming.coldStartBuffer.length, 1, "buffer must survive into connect()");

  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));
  await connected;

  streaming.sendAudio(Buffer.from("live"));
  const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);
  assert.deepEqual(payloads, [
    Buffer.from("pre-token-fetch audio").toString("base64"),
    Buffer.from("live").toString("base64"),
  ]);
  streaming.cleanup();
});

test("sendAudio upsamples 16kHz capture to the 24kHz session rate", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.inputRate = 24000;
  streaming.captureRate = 16000;
  streaming.ws = makeFakeSocket(WS.OPEN);

  const pcm = new Int16Array([0, 100, 200, 300]);
  streaming.sendAudio(Buffer.from(pcm.buffer));

  const raw = Buffer.from(JSON.parse(streaming.ws.sent[0]).audio, "base64");
  const out = new Int16Array(raw.buffer, raw.byteOffset, raw.length / 2);
  assert.deepEqual([...out], [0, 67, 133, 200, 267, 300]);
  assert.equal(streaming.audioBytesSent, out.length * 2);
});

test("cleanup() resets bufferingAudio so a dead instance stops buffering", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.beginConnecting();

  streaming.cleanup();

  assert.equal(streaming.bufferingAudio, false);
  const sent = streaming.sendAudio(Buffer.from([1, 2, 3]));
  assert.equal(sent, false);
  assert.equal(streaming.coldStartBuffer.length, 0);
});

test("cleanup() stops the keep-alive interval", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.ws = makeFakeSocket(WS.OPEN);
  streaming.startKeepAlive();

  assert.notEqual(streaming.keepAliveInterval, null);
  streaming.cleanup();
  assert.equal(streaming.keepAliveInterval, null);
});

test("keep-alive terminates a connection that misses a pong", (t) => {
  t.mock.timers.enable({ apis: ["setInterval"] });
  return (async () => {
    const OpenAIRealtimeStreaming = (await load()).default;
    const streaming = new OpenAIRealtimeStreaming();
    const socket = makeFakeSocket(WS.OPEN);
    let terminated = false;
    socket.terminate = () => {
      terminated = true;
      socket.readyState = WS.CLOSED;
    };
    streaming.ws = socket;

    streaming.startKeepAlive();

    t.mock.timers.tick(15000); // first tick: sends a ping, no pong arrives
    assert.equal(terminated, false);

    t.mock.timers.tick(15000); // second tick: no pong was received since the first ping
    assert.equal(terminated, true);
  })();
});

test("keep-alive stays alive when a pong is received between pings", (t) => {
  t.mock.timers.enable({ apis: ["setInterval"] });
  return (async () => {
    const OpenAIRealtimeStreaming = (await load()).default;
    const streaming = new OpenAIRealtimeStreaming();
    const socket = makeFakeSocket(WS.OPEN);
    let terminated = false;
    socket.terminate = () => {
      terminated = true;
    };
    streaming.ws = socket;

    streaming.startKeepAlive();

    t.mock.timers.tick(15000);
    socket.emit("pong");
    t.mock.timers.tick(15000);

    assert.equal(terminated, false);
  })();
});

// -- session expiry (60-minute OpenAI Realtime session limit) --

test("session_expired error fires onSessionExpired instead of onError and sets the flag", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  let expiredCalled = false;
  let errorCalled = false;
  streaming.onSessionExpired = () => {
    expiredCalled = true;
  };
  streaming.onError = () => {
    errorCalled = true;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "error",
      error: { code: "session_expired", message: "Your session hit the maximum duration." },
    })
  );

  assert.equal(expiredCalled, true);
  assert.equal(errorCalled, false);
  assert.equal(streaming._sessionExpired, true);
});

test("session_expired without an onSessionExpired handler falls through to onError (dictation path)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  let errorMessage = null;
  streaming.onError = (err) => {
    errorMessage = err.message;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "error",
      error: { code: "session_expired", message: "Your session hit the maximum duration." },
    })
  );

  assert.equal(errorMessage, "Your session hit the maximum duration.");
  assert.equal(streaming._sessionExpired, false);
});

test("non-session_expired error fires onError normally", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  let errorMsg = null;
  streaming.onError = (err) => {
    errorMsg = err.message;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "error",
      error: { code: "server_error", message: "something broke" },
    })
  );

  assert.equal(errorMsg, "something broke");
});

test("empty buffer error is not treated as session expiry", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  let expiredCalled = false;
  let errorCalled = false;
  streaming.onSessionExpired = () => {
    expiredCalled = true;
  };
  streaming.onError = () => {
    errorCalled = true;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "error",
      error: { code: "input_audio_buffer_commit_empty", message: "buffer too small" },
    })
  );

  assert.equal(expiredCalled, false);
  assert.equal(errorCalled, true);
});

test("close after session_expired does not fire onSessionEnd (reconnect owns the session)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectPreconfigured(streaming, socket);

  let sessionEndCalled = false;
  streaming.onSessionEnd = () => {
    sessionEndCalled = true;
  };
  streaming.onSessionExpired = () => {};

  socket.emit(
    "message",
    JSON.stringify({ type: "error", error: { code: "session_expired", message: "expired" } })
  );
  socket.emit("close", 1000, Buffer.from(""));

  assert.equal(sessionEndCalled, false);
});

test("connect() resets _sessionExpired left over from a previous session", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming._sessionExpired = true;

  const socket = makeFakeSocket(WS.CONNECTING);
  const connected = streaming.connect({
    apiKey: "key",
    preconfigured: true,
    createSocket: async () => socket,
  });
  assert.equal(streaming._sessionExpired, false, "flag must reset synchronously in connect()");

  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));
  await connected;
  streaming.cleanup();
});

// -- proactive session timer (fires before the 60-minute limit) --

test("session timer fires onSessionExpired at the pre-empt deadline while connected", (t) => {
  t.mock.timers.enable({ apis: ["setTimeout"] });
  return (async () => {
    const OpenAIRealtimeStreaming = (await load()).default;
    const streaming = new OpenAIRealtimeStreaming();
    streaming.isConnected = true;
    let expiredCalls = 0;
    streaming.onSessionExpired = () => {
      expiredCalls += 1;
    };

    streaming._startSessionTimer();

    t.mock.timers.tick(55 * 60 * 1000 - 1);
    assert.equal(expiredCalls, 0);
    t.mock.timers.tick(1);
    assert.equal(expiredCalls, 1);
  })();
});

test("cleanup() clears the session timer so a stopped session never requests a reconnect", (t) => {
  t.mock.timers.enable({ apis: ["setTimeout"] });
  return (async () => {
    const OpenAIRealtimeStreaming = (await load()).default;
    const streaming = new OpenAIRealtimeStreaming();
    streaming.isConnected = true;
    let expiredCalls = 0;
    streaming.onSessionExpired = () => {
      expiredCalls += 1;
    };

    streaming._startSessionTimer();
    streaming.cleanup();
    assert.equal(streaming._sessionTimer, null);

    t.mock.timers.tick(55 * 60 * 1000);
    assert.equal(expiredCalls, 0);
  })();
});

// -- reconnect flow: audio accounting across the session boundary --

test("zero audio loss during reactive reconnect (session_expired at 60min)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const CHUNK = Buffer.alloc(480);

  // Phase 1: old instance streaming normally.
  const old = new OpenAIRealtimeStreaming();
  old.isConnected = true;
  old.ws = makeFakeSocket(WS.OPEN);
  for (let i = 0; i < 100; i++) old.sendAudio(CHUNK);
  assert.equal(old.ws.sent.length, 100, "all chunks sent to old ws");
  const oldSent = old.ws.sent;

  // Phase 2: session_expired fires, server closes the old ws.
  let expiredFired = false;
  old.onSessionExpired = () => {
    expiredFired = true;
  };
  old.handleMessage(
    JSON.stringify({ type: "error", error: { code: "session_expired", message: "60 minutes" } })
  );
  assert.equal(expiredFired, true);
  old.cleanup();
  assert.equal(
    old.sendAudio(CHUNK),
    false,
    "dead instance drops audio, but the swap already happened"
  );

  // Phase 3: reconnect swaps in a fresh instance before the token fetch;
  // beginConnecting() arms the pre-connect buffer for the fetch window.
  const fresh = new OpenAIRealtimeStreaming();
  fresh.beginConnecting();
  for (let i = 0; i < 50; i++) fresh.sendAudio(CHUNK);
  assert.equal(fresh.coldStartBuffer.length, 50, "pre-connect buffer caught all chunks");
  assert.equal(fresh.coldStartBufferSize, 50 * 480);

  // Phase 4: token received, socket still connecting.
  fresh.ws = makeFakeSocket(WS.CONNECTING);
  for (let i = 0; i < 20; i++) fresh.sendAudio(CHUNK);
  assert.equal(fresh.coldStartBuffer.length, 70, "buffer holds pre-connect + connecting chunks");

  // Phase 5: socket opens, next sendAudio flushes everything.
  fresh.ws = makeFakeSocket(WS.OPEN);
  fresh.sendAudio(CHUNK);
  assert.equal(fresh.coldStartBuffer.length, 0, "buffer flushed");
  assert.equal(fresh.ws.sent.length, 71, "all buffered + live chunks sent to new ws");

  assert.equal(oldSent.length + fresh.ws.sent.length, 171, "zero chunks dropped");
});

test("zero audio loss during proactive reconnect (timer before the limit)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const CHUNK = Buffer.alloc(480);

  // Old instance still alive while the reconnect fetches a token.
  const old = new OpenAIRealtimeStreaming();
  old.isConnected = true;
  old.ws = makeFakeSocket(WS.OPEN);
  for (let i = 0; i < 100; i++) old.sendAudio(CHUNK);

  // Audio dispatched between the timer firing and the instance swap still
  // reaches the old, healthy connection.
  for (let i = 0; i < 10; i++) old.sendAudio(CHUNK);
  assert.equal(old.ws.sent.length, 110, "audio still flows to old instance during token fetch");

  // References swapped; new instance buffers until its socket opens.
  const fresh = new OpenAIRealtimeStreaming();
  fresh.beginConnecting();
  for (let i = 0; i < 30; i++) fresh.sendAudio(CHUNK);
  assert.equal(fresh.coldStartBuffer.length, 30);

  fresh.ws = makeFakeSocket(WS.OPEN);
  fresh.sendAudio(CHUNK);
  assert.equal(fresh.ws.sent.length, 31, "30 flushed + 1 live");
  assert.equal(fresh.coldStartBuffer.length, 0);

  assert.equal(old.ws.sent.length + fresh.ws.sent.length, 141, "zero chunks dropped");
});

test("concurrent session_expired from mic and system streams only reconnects once", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;

  // Mirrors the meetingReconnecting guard in ipcHandlers.
  let reconnectCalls = 0;
  let reconnecting = false;
  const guardedReconnect = () => {
    if (reconnecting) return;
    reconnecting = true;
    reconnectCalls++;
  };

  const mic = new OpenAIRealtimeStreaming();
  const system = new OpenAIRealtimeStreaming();
  mic.onSessionExpired = guardedReconnect;
  system.onSessionExpired = guardedReconnect;

  const expiredEvent = JSON.stringify({
    type: "error",
    error: { code: "session_expired", message: "expired" },
  });
  mic.handleMessage(expiredEvent);
  system.handleMessage(expiredEvent);

  assert.equal(reconnectCalls, 1, "reconnect called exactly once despite two streams expiring");
});

test("completedSegments accumulate across turns", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  let lastFull = "";
  streaming.onFinalTranscript = (text) => {
    lastFull = text;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.completed",
      transcript: "Hello world",
    })
  );
  streaming.handleMessage(
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.completed",
      transcript: "How are you",
    })
  );

  assert.equal(streaming.completedSegments.length, 2);
  assert.equal(lastFull, "Hello world How are you");
});

test("BYOK session.update is byte-for-byte today's payload when no VAD/language/noise options are passed", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);

  const connected = streaming.connect({
    apiKey: "sk-test",
    model: "gpt-4o-mini-transcribe",
    // keyterms stay ignored: the dictionary has no realtime channel yet.
    keyterms: ["OpenWhispr"],
    sampleRate: 24000,
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));

  const updates = socket.sent
    .map((raw) => JSON.parse(raw))
    .filter((e) => e.type === "session.update");
  assert.equal(updates.length, 1, "exactly one session.update after session.created");
  assert.deepEqual(updates[0], {
    type: "session.update",
    session: {
      type: "transcription",
      audio: {
        input: {
          format: { type: "audio/pcm", rate: 24000 },
          transcription: { model: "gpt-4o-mini-transcribe" },
          turn_detection: {
            type: "server_vad",
            threshold: 0.6,
            silence_duration_ms: 600,
            prefix_padding_ms: 500,
          },
        },
      },
    },
  });

  socket.emit("message", JSON.stringify({ type: "session.updated" }));
  await connected;
  streaming.cleanup();
});

// -- #2050: the dictation language reaches the BYOK transcription config --

test("BYOK session.update pins the dictation language the caller selected (#2050)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe", language: "de" });

  const [update] = sentEvents(socket, "session.update");
  assert.deepEqual(update.session.audio.input.transcription, {
    model: "gpt-4o-mini-transcribe",
    language: "de",
  });
  streaming.cleanup();
});

test("BYOK session.update sends the base ISO-639-1 code and omits language for auto/blank", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const cases = [
    ["zh-TW", "zh"],
    [" PT ", "pt"],
    ["auto", undefined],
    ["", undefined],
    [undefined, undefined],
  ];
  for (const [language, expected] of cases) {
    const streaming = new OpenAIRealtimeStreaming();
    const socket = makeFakeSocket(WS.CONNECTING);
    await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe", language });

    const [update] = sentEvents(socket, "session.update");
    assert.equal(update.session.audio.input.transcription.language, expected, String(language));
    assert.equal(
      "language" in update.session.audio.input.transcription,
      expected !== undefined,
      `${String(language)} must not leave a language key on the wire`
    );
    streaming.cleanup();
  }
});

test("a later connect() without a language does not inherit the previous session's pin", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const first = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, first, { model: "gpt-4o-mini-transcribe", language: "de" });
  streaming.cleanup();

  const second = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, second, { model: "gpt-4o-mini-transcribe" });
  const [update] = sentEvents(second, "session.update");
  assert.equal("language" in update.session.audio.input.transcription, false);
  streaming.cleanup();
});

test("BYOK session.update is byte-for-byte the overridden payload when vadThreshold is passed (system socket)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);

  const connected = streaming.connect({
    apiKey: "sk-test",
    model: "gpt-4o-mini-transcribe",
    streamLabel: "system",
    vadThreshold: 0.3,
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));

  const updates = socket.sent
    .map((raw) => JSON.parse(raw))
    .filter((e) => e.type === "session.update");
  assert.equal(updates.length, 1, "exactly one session.update after session.created");
  assert.deepEqual(updates[0], {
    type: "session.update",
    session: {
      type: "transcription",
      audio: {
        input: {
          format: { type: "audio/pcm", rate: 24000 },
          transcription: { model: "gpt-4o-mini-transcribe" },
          turn_detection: {
            type: "server_vad",
            threshold: 0.3,
            silence_duration_ms: 600,
            prefix_padding_ms: 500,
          },
        },
      },
    },
  });

  socket.emit("message", JSON.stringify({ type: "session.updated" }));
  await connected;
  streaming.cleanup();
});

test("dictation-style connect (inputRate 16000, custom socket factory) declares the 16kHz format and the same VAD", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  const factoryCalls = [];

  const connected = streaming.connect({
    apiKey: "tk-secret",
    model: "voxtral-mini-4b-realtime",
    inputRate: 16000,
    createSocket: () => {
      factoryCalls.push("called");
      return Promise.resolve(socket);
    },
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));

  const [update] = socket.sent
    .map((raw) => JSON.parse(raw))
    .filter((e) => e.type === "session.update");
  assert.equal(factoryCalls.length, 1);
  assert.deepEqual(update.session.audio.input.format, { type: "audio/pcm", rate: 16000 });
  assert.equal(update.session.audio.input.transcription.model, "voxtral-mini-4b-realtime");
  assert.deepEqual(update.session.audio.input.turn_detection, {
    type: "server_vad",
    threshold: 0.6,
    silence_duration_ms: 600,
    prefix_padding_ms: 500,
  });
  assert.equal("noise_reduction" in update.session.audio.input, false);
  assert.equal("language" in update.session.audio.input.transcription, false);

  socket.emit("message", JSON.stringify({ type: "session.updated" }));
  await connected;
  streaming.cleanup();
});

test("a final that lands during disconnect()'s commit window reaches onFinalTranscript with its timestamp", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectPreconfigured(streaming, socket);

  const calls = [];
  streaming.onFinalTranscript = (text, timestamp) => calls.push({ text, timestamp });
  streaming.sendAudio(Buffer.alloc(1600, 1)); // audioBytesSent > 0 so disconnect() commits

  const disconnecting = streaming.disconnect();
  await new Promise((resolve) => setImmediate(resolve));

  const commits = socket.sent
    .filter((raw) => typeof raw === "string")
    .map((raw) => JSON.parse(raw))
    .filter((e) => e.type === "input_audio_buffer.commit");
  assert.equal(commits.length, 1, "disconnect() commits the trailing buffer");

  socket.emit("message", JSON.stringify({ type: "input_audio_buffer.speech_started" }));
  socket.emit(
    "message",
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.completed",
      transcript: "tail words",
    })
  );
  const result = await disconnecting;

  assert.equal(result.text, "tail words", "the tail is returned to the caller");
  assert.equal(calls.length, 1, "the tail also reaches the live onFinalTranscript handler");
  assert.equal(calls[0].text, "tail words");
  assert.equal(typeof calls[0].timestamp, "number");
});

// -- diagnostic logging: stream labels, VAD events, failed/empty turns --

test("transcription.failed is logged with the stream label and stays log-only (no onError, no segment)", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.streamLabel = "system";
  const logs = captureLogs(t);
  let errorCalled = false;
  streaming.onError = () => {
    errorCalled = true;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.failed",
      item_id: "item_1",
      error: { code: "audio_unintelligible", message: "audio too noisy" },
    })
  );

  const line = logs.find((entry) => entry.message.includes("turn transcription failed"));
  assert.ok(line, "failed turn must produce a debug line");
  assert.equal(line.level, "debug");
  assert.equal(line.meta.stream, "system");
  assert.equal(line.meta.itemId, "item_1");
  assert.equal(line.meta.error, "audio too noisy");
  assert.equal(errorCalled, false, "failed turns must not surface as errors");
  assert.equal(streaming.completedSegments.length, 0, "failed turns must not count as segments");
});

test("empty-transcript completed events are logged and add no segment", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.streamLabel = "system";
  streaming.audioBytesSent = 4800;
  const logs = captureLogs(t);
  let finalCalled = false;
  streaming.onFinalTranscript = () => {
    finalCalled = true;
  };

  streaming.handleMessage(
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "item_2",
      transcript: "   ",
    })
  );

  const line = logs.find((entry) => entry.message.includes("turn completed with empty transcript"));
  assert.ok(line, "empty completion must produce a debug line");
  assert.equal(line.meta.stream, "system");
  assert.equal(line.meta.itemId, "item_2");
  assert.equal(line.meta.audioBytesSent, 4800);
  assert.equal(finalCalled, false);
  assert.equal(streaming.completedSegments.length, 0);
});

test("VAD events log first-3-then-every-50th with streamLabel and audioBytesSent", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  streaming.streamLabel = "system";
  streaming.audioBytesSent = 1234;
  const logs = captureLogs(t);
  const vadLines = () => logs.filter((entry) => entry.message.includes("VAD event"));

  streaming.handleMessage(JSON.stringify({ type: "input_audio_buffer.speech_started" }));
  streaming.handleMessage(JSON.stringify({ type: "input_audio_buffer.speech_stopped" }));
  streaming.handleMessage(JSON.stringify({ type: "input_audio_buffer.committed" }));
  assert.equal(vadLines().length, 3, "first 3 events all log");
  assert.equal(vadLines()[0].meta.stream, "system");
  assert.equal(vadLines()[0].meta.event, "input_audio_buffer.speech_started");
  assert.equal(vadLines()[0].meta.audioBytesSent, 1234);

  // Events 4..49 are throttled; the 50th logs again.
  for (let i = 4; i <= 49; i++) {
    streaming.handleMessage(JSON.stringify({ type: "input_audio_buffer.committed" }));
  }
  assert.equal(vadLines().length, 3, "events 4..49 stay silent");
  streaming.handleMessage(JSON.stringify({ type: "input_audio_buffer.speech_started" }));
  assert.equal(vadLines().length, 4, "the 50th event logs");
  assert.equal(vadLines()[3].meta.count, 50);

  assert.equal(streaming.speechStartedCount, 2, "only speech_started increments the counter");
});

test("disconnect summary carries streamLabel and speechStartedCount", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  const connected = streaming.connect({
    apiKey: "key",
    preconfigured: true,
    streamLabel: "system",
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit("message", JSON.stringify({ type: "session.created" }));
  await connected;

  socket.emit("message", JSON.stringify({ type: "input_audio_buffer.speech_started" }));
  socket.emit("message", JSON.stringify({ type: "input_audio_buffer.speech_started" }));

  const logs = captureLogs(t);
  await streaming.disconnect();

  const line = logs.find((entry) => entry.message === "OpenAI Realtime disconnect");
  assert.ok(line, "disconnect must log a summary");
  assert.equal(line.meta.stream, "system");
  assert.equal(line.meta.speechStartedCount, 2);
  assert.equal(line.meta.segments, 0);
  assert.equal(line.meta.audioBytesSent, 0);
});

test("preconfigured session.created echoes the server VAD config and format into the log", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  const logs = captureLogs(t);

  const connected = streaming.connect({
    apiKey: "key",
    preconfigured: true,
    streamLabel: "system",
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit(
    "message",
    JSON.stringify({
      type: "session.created",
      session: {
        audio: {
          input: {
            format: { type: "audio/pcm", rate: 24000 },
            turn_detection: { type: "server_vad", threshold: 0.5, silence_duration_ms: 600 },
          },
        },
      },
    })
  );
  await connected;

  const lines = logs.filter((entry) => entry.message.includes("session created (preconfigured)"));
  assert.equal(lines.length, 1, "server config echo logs once per socket");
  assert.equal(lines[0].meta.stream, "system");
  assert.deepEqual(lines[0].meta.turnDetection, {
    type: "server_vad",
    threshold: 0.5,
    silence_duration_ms: 600,
  });
  assert.deepEqual(lines[0].meta.inputFormat, { type: "audio/pcm", rate: 24000 });
  streaming.cleanup();
});

test("without a streamLabel (dictation) log metadata carries no stream key", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const logs = captureLogs(t);

  await streaming.disconnect();

  const line = logs.find((entry) => entry.message === "OpenAI Realtime disconnect");
  assert.ok(line);
  assert.equal("stream" in line.meta, false, "unlabelled sockets keep today's metadata shape");
});

// -- gpt-live-transcribe: no server VAD, the client commits the turn on stop --

const LIVE_UPDATE_BODY = {
  type: "session.update",
  session: {
    type: "transcription",
    audio: {
      input: {
        format: { type: "audio/pcm", rate: 24000 },
        transcription: { model: "gpt-live-transcribe" },
        turn_detection: null,
      },
    },
  },
};

const completedEvent = (transcript) =>
  JSON.stringify({
    type: "conversation.item.input_audio_transcription.completed",
    item_id: "item_1",
    transcript,
  });

test("gpt-live-transcribe session.update sends turn_detection: null (any VAD config is rejected); the rest is the legacy payload", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-live-transcribe", vadThreshold: 0.3 });

  const updates = sentEvents(socket, "session.update");
  assert.equal(updates.length, 1);
  assert.deepEqual(updates[0], LIVE_UPDATE_BODY);
  assert.ok(socket.sent[0].includes('"turn_detection":null'), "null is on the wire, not dropped");
  streaming.cleanup();
});

test("legacy models keep the server_vad block byte-for-byte", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  for (const model of ["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]) {
    const streaming = new OpenAIRealtimeStreaming();
    const socket = makeFakeSocket(WS.CONNECTING);
    await connectByok(streaming, socket, { model });

    const [update] = sentEvents(socket, "session.update");
    assert.equal(update.session.audio.input.transcription.model, model);
    assert.deepEqual(
      update.session.audio.input.turn_detection,
      { type: "server_vad", threshold: 0.6, silence_duration_ms: 600, prefix_padding_ms: 500 },
      model
    );
    streaming.cleanup();
  }
});

test("gpt-live-transcribe stop: disconnect() commits once, then consumes the .completed final it triggered", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
  const partials = [];
  const finals = [];
  streaming.onPartialTranscript = (text) => partials.push(text);
  streaming.onFinalTranscript = (text, timestamp) => finals.push({ text, timestamp });

  // Partials stream during speech with no speech_started/speech_stopped around them.
  for (const delta of ["Hello", " world"]) {
    socket.emit(
      "message",
      JSON.stringify({ type: "conversation.item.input_audio_transcription.delta", delta })
    );
  }
  streaming.sendAudio(Buffer.alloc(4800, 1));

  const disconnecting = streaming.disconnect();
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 1);
  assert.equal(finals.length, 0, "the final only exists once the server answers the commit");

  socket.emit(
    "message",
    JSON.stringify({ type: "input_audio_buffer.committed", item_id: "item_1" })
  );
  socket.emit("message", completedEvent("Hello world."));
  const result = await disconnecting;

  assert.deepEqual(partials, ["Hello", "Hello world"]);
  assert.equal(result.text, "Hello world.");
  assert.deepEqual(
    finals.map((f) => f.text),
    ["Hello world."]
  );
  assert.equal(typeof finals[0].timestamp, "number", "timestamp falls back without speech_started");
  assert.equal(streaming.speechStartedCount, 0);
  assert.equal(
    sentEvents(socket, "input_audio_buffer.commit").length,
    1,
    "still exactly one commit"
  );
  assert.equal(socket.readyState, WS.CLOSED);
});

test("gpt-live-transcribe stop with no audio sent never commits (an empty commit is a server error)", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-live-transcribe" });

  const result = await streaming.disconnect();

  assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 0);
  assert.equal(result.text, "");
  assert.equal(socket.readyState, WS.CLOSED);
});

test("gpt-live-transcribe stop outlives the legacy 3 s commit budget; legacy keeps it", (t) => {
  t.mock.timers.enable({ apis: ["setTimeout"] });
  return (async () => {
    const OpenAIRealtimeStreaming = (await load()).default;
    const settle = async () => new Promise((resolve) => setImmediate(resolve));

    const live = new OpenAIRealtimeStreaming();
    const liveSocket = makeFakeSocket(WS.CONNECTING);
    await connectByok(live, liveSocket, { model: "gpt-live-transcribe" });
    live.sendAudio(Buffer.alloc(4800, 1));
    let liveSettled = false;
    const liveDisconnect = live.disconnect().then((result) => {
      liveSettled = true;
      return result;
    });
    await settle();
    t.mock.timers.tick(3000);
    await settle();
    assert.equal(liveSettled, false, "a long dictation's final trails stop by more than 3 s");
    liveSocket.emit("message", completedEvent("late tail"));
    assert.equal((await liveDisconnect).text, "late tail");

    const legacy = new OpenAIRealtimeStreaming();
    const legacySocket = makeFakeSocket(WS.CONNECTING);
    await connectByok(legacy, legacySocket, { model: "gpt-4o-mini-transcribe" });
    legacy.sendAudio(Buffer.alloc(4800, 1));
    let legacySettled = false;
    const legacyDisconnect = legacy.disconnect().then(() => {
      legacySettled = true;
    });
    await settle();
    t.mock.timers.tick(3000);
    await legacyDisconnect;
    assert.equal(legacySettled, true);
  })();
});

test("an empty or failed committed turn ends the stop wait at once instead of stalling for the budget", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const terminalEvents = [
    completedEvent("   "),
    JSON.stringify({
      type: "conversation.item.input_audio_transcription.failed",
      item_id: "item_1",
      error: { code: "audio_unintelligible", message: "audio too noisy" },
    }),
  ];
  for (const terminal of terminalEvents) {
    const streaming = new OpenAIRealtimeStreaming();
    const socket = makeFakeSocket(WS.CONNECTING);
    await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
    let finalCalled = false;
    streaming.onFinalTranscript = () => {
      finalCalled = true;
    };
    streaming.sendAudio(Buffer.alloc(4800, 1));

    const disconnecting = streaming.disconnect();
    await new Promise((resolve) => setImmediate(resolve));
    socket.emit("message", terminal);
    const result = await Promise.race([
      disconnecting,
      new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),
    ]);

    assert.equal(result.text, "");
    assert.equal(finalCalled, false);
    assert.equal(streaming._onTurnSettled, null, "hook is released once the wait ends");
  }
});

test("a socket that closes during the commit wait ends the stop at once with the accumulated text", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
  streaming.sendAudio(Buffer.alloc(4800, 1));

  const disconnecting = streaming.disconnect();
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.CLOSED;
  socket.emit("close", 1006, Buffer.from(""));
  const result = await Promise.race([
    disconnecting,
    new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),
  ]);

  assert.equal(result.text, "");
  assert.equal(streaming.ws, null);
});

test("preconfigured sessions adopt the model the server minted (session.created echo) and never send session.update", async (t) => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const logs = captureLogs(t);
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);

  const connected = streaming.connect({
    apiKey: "cs-secret",
    preconfigured: true,
    // The desktop's BYOK default; the server decides the managed model and
    // pins the language when it mints the secret.
    model: "gpt-4o-mini-transcribe",
    language: "de",
    createSocket: async () => socket,
  });
  await new Promise((resolve) => setImmediate(resolve));
  socket.readyState = WS.OPEN;
  socket.emit(
    "message",
    JSON.stringify({
      type: "session.created",
      session: {
        audio: {
          input: {
            format: { type: "audio/pcm", rate: 24000 },
            transcription: { model: "gpt-live-transcribe", language: null },
            turn_detection: null,
          },
        },
      },
    })
  );
  await connected;

  assert.equal(streaming.model, "gpt-live-transcribe");
  assert.equal(sentEvents(socket, "session.update").length, 0);
  const line = logs.find((entry) => entry.message.includes("session created (preconfigured)"));
  assert.equal(line.meta.model, "gpt-live-transcribe");
  assert.equal(line.meta.turnDetection, null);
  streaming.cleanup();

  // No echo (older servers, bare mocks): the caller's model stands.
  const bare = new OpenAIRealtimeStreaming();
  await connectPreconfigured(bare, makeFakeSocket(WS.CONNECTING));
  assert.equal(bare.model, "gpt-4o-mini-transcribe");
  bare.cleanup();
});

test("stop keeps the socket open until every committed VAD turn settles", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
  const emit = (event) => socket.emit("message", JSON.stringify(event));
  streaming.sendAudio(Buffer.alloc(4800, 1));
  emit({ type: "input_audio_buffer.committed", item_id: "first" });
  const stopping = streaming.disconnect();
  emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "first" });
  emit({
    type: "conversation.item.input_audio_transcription.completed",
    item_id: "first",
    transcript: "First sentence.",
  });
  await new Promise((resolve) => setImmediate(resolve));
  try {
    assert.equal(socket.readyState, WS.OPEN, "the earlier turn cannot close out the tail");
    emit({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "tail",
      transcript: "Last sentence.",
    });
    assert.equal((await stopping).text, "First sentence. Last sentence.");
  } finally {
    streaming.cleanup();
    await stopping;
  }
});

test("an empty stop commit still waits for an already committed VAD transcript", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
  const emit = (event) => socket.emit("message", JSON.stringify(event));
  streaming.sendAudio(Buffer.alloc(4800, 1));
  emit({ type: "input_audio_buffer.committed", item_id: "vad-turn" });
  const stopping = streaming.disconnect();
  emit({
    type: "error",
    error: { code: "input_audio_buffer_commit_empty", message: "buffer too small" },
  });
  await new Promise((resolve) => setImmediate(resolve));
  try {
    assert.equal(socket.readyState, WS.OPEN, "an empty buffer says nothing about pending ASR");
    emit({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "vad-turn",
      transcript: "Already committed speech.",
    });
    assert.equal((await stopping).text, "Already committed speech.");
  } finally {
    streaming.cleanup();
    await stopping;
  }
});

test("a failed VAD turn settles only itself while another transcript is pending", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
  const emit = (event) => socket.emit("message", JSON.stringify(event));
  streaming.sendAudio(Buffer.alloc(4800, 1));
  emit({ type: "input_audio_buffer.committed", item_id: "noisy" });
  emit({ type: "input_audio_buffer.committed", item_id: "speech" });
  const stopping = streaming.disconnect();
  emit({
    type: "error",
    error: { code: "input_audio_buffer_commit_empty", message: "commit_empty" },
  });
  emit({
    type: "conversation.item.input_audio_transcription.failed",
    item_id: "noisy",
    error: { code: "audio_unintelligible", message: "audio too noisy" },
  });
  await new Promise((resolve) => setImmediate(resolve));
  try {
    assert.equal(socket.readyState, WS.OPEN, "a failed turn cannot discard another turn");
    emit({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "speech",
      transcript: "Keep this speech.",
    });
    assert.equal((await stopping).text, "Keep this speech.");
  } finally {
    streaming.cleanup();
    await stopping;
  }
});

test("an earlier final cannot finish stop before the trailing commit is acknowledged", async () => {
  const OpenAIRealtimeStreaming = (await load()).default;
  const streaming = new OpenAIRealtimeStreaming();
  const socket = makeFakeSocket(WS.CONNECTING);
  await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
  const emit = (event) => socket.emit("message", JSON.stringify(event));
  streaming.sendAudio(Buffer.alloc(4800, 1));
  emit({ type: "input_audio_buffer.committed", item_id: "earlier" });
  const stopping = streaming.disconnect();
  emit({
    type: "conversation.item.input_audio_transcription.completed",
    item_id: "earlier",
    transcript: "Earlier speech.",
  });
  await new Promise((resolve) => setImmediate(resolve));
  try {
    assert.equal(socket.readyState, WS.OPEN, "stop must still await its own commit response");
    emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "earlier" });
    emit({
      type: "conversation.item.input_audio_transcription.completed",
      item_id: "tail",
      transcript: "Trailing speech.",
    });
    assert.equal((await stopping).text, "Earlier speech. Trailing speech.");
  } finally {
    streaming.cleanup();
    await stopping;
  }
});