Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

16.1 KB
Raw Blame
import test from "node:test";
import assert from "node:assert/strict";

const {
  isUselessEmptyTurn,
  summarizeReplayedUpstreamTurn,
  readBoundedResponseText,
  readBoundedResponseOutcome,
  judgeBufferedTurn,
  FLUSH_EMPTY_RETRY_MAX_BYTES,
  pickEmptyTurnRetryCredentials,
  swapCredentialsInPlace,
} = await import("../../open-sse/utils/emptyTurnRetry.ts");
const { isEmptyTurnCore } = await import("../../open-sse/utils/streamEmptyChoices.ts");
const { FORMATS } = await import("../../open-sse/translator/formats.ts");

const base = {
  finishReason: "stop",
  contentText: "",
  reasoningText: "",
  forwardedValuableChunk: false,
  hasValidUsage: false,
  toolCallsPresent: false,
};

test("reasoning-only turn: stop + empty text + non-empty reasoning is an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, reasoningText: "some thinking trace" }), true);
});

test("zero-valuable-chunk turn: no valuable chunks and no valid usage is an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, finishReason: "" }), true);
});

test("non-empty content text is not an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, contentText: "hello" }), false);
});

test("valid usage alone is not an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, hasValidUsage: true }), false);
});

test("forwarded valuable chunk alone is not an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, forwardedValuableChunk: true }), false);
});

test("legit finish reasons are not empty turns", () => {
  for (const finishReason of ["length", "tool_calls", "content_filter"]) {
    assert.equal(isUselessEmptyTurn({ ...base, finishReason }), false);
  }
});

test("tool calls present are not an empty turn", () => {
  assert.equal(isUselessEmptyTurn({ ...base, toolCallsPresent: true }), false);
});

test("shared core matches the frozen guard condition", () => {
  assert.equal(isEmptyTurnCore(false, false), true);
  assert.equal(isEmptyTurnCore(true, false), false);
  assert.equal(isEmptyTurnCore(false, true), false);
});

test("memory cap is a positive bound", () => {
  assert.ok(FLUSH_EMPTY_RETRY_MAX_BYTES > 0);
});

function sse(...frames: string[]): string {
  return frames.map((f) => `data: ${f}\n\n`).join("") + "data: [DONE]\n\n";
}

const chatChunk = (delta: Record<string, unknown>, finish: string | null = null) =>
  JSON.stringify({
    id: "chatcmpl-probe",
    object: "chat.completion.chunk",
    model: "probe",
    choices: [{ delta, finish_reason: finish }],
  });

test("replay: reasoning-only chat turn classifies empty (parity with live transform)", () => {
  const body = sse(chatChunk({ reasoning_content: "thinking trace here" }), chatChunk({}, "stop"));
  const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI, FORMATS.OPENAI);
  assert.ok(summary, "replay must produce a summary");
  assert.equal(summary.reasoningText.length > 0, true);
  assert.equal(summary.contentText, "");
  assert.equal(isUselessEmptyTurn(summary), true);
});

test("replay: all-empty choices turn classifies empty", () => {
  const body = sse(
    JSON.stringify({
      id: "chatcmpl-probe",
      object: "chat.completion.chunk",
      model: "probe",
      choices: [],
    })
  );
  const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI, FORMATS.OPENAI);
  assert.ok(summary, "replay must produce a summary");
  assert.equal(isUselessEmptyTurn(summary), true);
});

test("replay: turn with real content classifies useful", () => {
  const body = sse(chatChunk({ content: "hello" }), chatChunk({}, "stop"));
  const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI, FORMATS.OPENAI);
  assert.ok(summary, "replay must produce a summary");
  assert.equal(isUselessEmptyTurn(summary), false);
});

test("replay: retry with content after empty first turn serves content", () => {
  const attempts = [
    sse(chatChunk({ reasoning_content: "only thinking" }), chatChunk({}, "stop")),
    sse(chatChunk({ content: "real answer" }), chatChunk({}, "stop")),
  ];
  let calls = 0;
  for (const body of attempts) {
    calls++;
    const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI, FORMATS.OPENAI);
    assert.ok(summary);
    if (calls === 1) {
      assert.equal(isUselessEmptyTurn(summary), true);
      continue;
    }
    assert.equal(isUselessEmptyTurn(summary), false);
  }
  assert.equal(calls, 2, "exactly 2 attempts: initial + 1 retry");
});

test("replay: retry also empty stays empty (fall back to current behavior)", () => {
  const body = sse(
    JSON.stringify({
      id: "chatcmpl-probe",
      object: "chat.completion.chunk",
      model: "probe",
      choices: [],
    })
  );
  for (let attempt = 0; attempt < 2; attempt++) {
    const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI, FORMATS.OPENAI);
    assert.ok(summary);
    assert.equal(isUselessEmptyTurn(summary), true);
  }
});

test("bounded read abandons past the cap without buffering everything", async () => {
  const big = new ReadableStream<Uint8Array>({
    start(controller) {
      controller.enqueue(
        new TextEncoder().encode("data: " + "x".repeat(FLUSH_EMPTY_RETRY_MAX_BYTES + 1) + "\n\n")
      );
      controller.close();
    },
  });
  const res = new Response(big, { status: 200 });
  const out = await readBoundedResponseText(res, FLUSH_EMPTY_RETRY_MAX_BYTES);
  assert.equal(out, null, "past-cap body must be abandoned, not buffered");
});

// #14691: a turn that already carries content or a tool call stops the bounded read at its
// first useful chunk (`early-pass`); only content-free turns are drained to the end. The
// "intact body" contract therefore applies to a content-free (reasoning-only) turn.
const contentFreeBody = () =>
  sse(chatChunk({ reasoning_content: "thinking" }), chatChunk({}, "stop"));

test("bounded read returns small bodies intact", async () => {
  const body = contentFreeBody();
  const res = new Response(body, { status: 200 });
  const out = await readBoundedResponseText(res, FLUSH_EMPTY_RETRY_MAX_BYTES);
  assert.equal(out, body);
});

test("bounded read outcome tells a read failure apart from an over-cap body", async () => {
  const failing = new ReadableStream<Uint8Array>({
    pull(controller) {
      controller.error(new TypeError("terminated"));
    },
  });
  const failed = await readBoundedResponseOutcome(
    new Response(failing, { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES
  );
  assert.equal(failed.kind, "error", "a stream that throws while being read is a failure");

  const big = new ReadableStream<Uint8Array>({
    start(controller) {
      controller.enqueue(new TextEncoder().encode("x".repeat(FLUSH_EMPTY_RETRY_MAX_BYTES + 1)));
      controller.close();
    },
  });
  const skipped = await readBoundedResponseOutcome(
    new Response(big, { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES
  );
  assert.equal(skipped.kind, "skipped", "an over-cap body is passed through, not retried");

  const body = contentFreeBody();
  const ok = await readBoundedResponseOutcome(
    new Response(body, { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES
  );
  assert.deepEqual(ok, { kind: "text", text: body });

  const useful = await readBoundedResponseOutcome(
    new Response(sse(chatChunk({ content: "hi" })), { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES
  );
  assert.deepEqual(
    useful,
    { kind: "early-pass" },
    "a content turn stops at its first useful chunk"
  );
});

test("bounded read returns skipped without awaiting a tee clone cancel", async () => {
  const BUDGET_MS = 2000;
  const neverClosing = new ReadableStream<Uint8Array>({
    start(controller) {
      controller.enqueue(new TextEncoder().encode("x".repeat(FLUSH_EMPTY_RETRY_MAX_BYTES + 1)));
    },
  });
  let res: Response | undefined;
  const start = Date.now();
  let timer: ReturnType<typeof setTimeout> | undefined;
  try {
    res = new Response(neverClosing, { status: 200 });
    const outcome = await Promise.race([
      readBoundedResponseOutcome(res, FLUSH_EMPTY_RETRY_MAX_BYTES),
      new Promise<never>((_, rej) => {
        timer = setTimeout(() => rej(new Error("bounded read timed out")), BUDGET_MS);
      }),
    ]);
    assert.equal(outcome.kind, "skipped");
    assert.ok(Date.now() - start < BUDGET_MS, `expected bounded skipped well under ${BUDGET_MS}ms`);
  } finally {
    if (timer !== undefined) clearTimeout(timer);
    await res?.body?.cancel().catch(() => undefined);
  }
});

test("buffered turn verdict: a dropped stream retries unless the client went away", () => {
  const dropped = { kind: "error" } as const;
  const retry = judgeBufferedTurn(dropped, FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, false);
  assert.equal(retry.kind, "retry");
  const gone = judgeBufferedTurn(dropped, FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, true);
  assert.equal(gone.kind, "pass", "a client that disconnected must not cost another upstream call");
  const skipped = judgeBufferedTurn(
    { kind: "skipped" },
    FORMATS.OPENAI_RESPONSES,
    FORMATS.OPENAI,
    false
  );
  assert.equal(skipped.kind, "pass", "an unclassifiable turn passes through");
});

test("replay: Responses reasoning-only deltas without completed classify empty", () => {
  const body = [
    `data: ${JSON.stringify({ type: "response.reasoning_text.delta", delta: "thinking here" })}\n\n`,
    `data: ${JSON.stringify({ type: "response.reasoning_summary_text.done", text: "thinking here" })}\n\n`,
  ].join("");
  const summary = summarizeReplayedUpstreamTurn(body, FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI);
  assert.ok(summary, "replay must produce a summary");
  assert.equal(isUselessEmptyTurn(summary), true);
});

test("a stalled stream with nothing usable is retried, not surfaced as an error", async () => {
  // An upstream that answers, sends a keepalive, then goes silent without ever
  // closing or erroring: buffering can never end on its own.
  const silent = new ReadableStream<Uint8Array>({
    start(controller) {
      controller.enqueue(new TextEncoder().encode(": keepalive\n\n"));
    },
  });
  const started = Date.now();
  const out = await readBoundedResponseOutcome(
    new Response(silent, { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES,
    50
  );
  assert.equal(out.kind, "idle", "a stream that stops producing must end the buffered read");
  assert.ok(Date.now() - started < 2000, "the read must return on its own budget");

  const verdict = judgeBufferedTurn(out, FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, false);
  assert.equal(verdict.kind, "retry", "nothing usable was produced: replay, never a bare failure");

  const gone = judgeBufferedTurn(out, FORMATS.OPENAI_RESPONSES, FORMATS.OPENAI, true);
  assert.equal(gone.kind, "pass", "a client that disconnected must not cost another dispatch");
});

test("a stalled stream that already carries content is passed through, not replayed", async () => {
  const partial = new ReadableStream<Uint8Array>({
    start(controller) {
      controller.enqueue(new TextEncoder().encode(sse(chatChunk({ content: "hello" }))));
    },
  });
  const out = await readBoundedResponseOutcome(
    new Response(partial, { status: 200 }),
    FLUSH_EMPTY_RETRY_MAX_BYTES,
    50
  );
  // #14691: the content chunk ends the read before the stall is ever observed.
  assert.equal(out.kind, "early-pass");
  const verdict = judgeBufferedTurn(out, FORMATS.OPENAI, FORMATS.OPENAI, false);
  assert.equal(verdict.kind, "pass", "content already produced is worth keeping");
});

test("bounded read keeps no budget when the idle budget is zero", async () => {
  const body = contentFreeBody();
  const out = await readBoundedResponseOutcome(new Response(body, { status: 200 }), 256_000, 0);
  assert.deepEqual(out, { kind: "text", text: body }, "a disabled budget must not change reads");
});

function recordingSelector(results: Record<string, unknown>) {
  const calls: Array<{ exclude: string | null; allowed: string[] | null }> = [];
  const select = async (
    _provider: string,
    excludeConnectionId: string | null,
    allowedConnections: string[] | null,
    _model: string | null
  ) => {
    calls.push({ exclude: excludeConnectionId, allowed: allowedConnections });
    return results[String(excludeConnectionId)] ?? null;
  };
  return { calls, select };
}

const unconstrained = { leased: false, forcedConnectionId: null, apiKey: null };

test("retry credentials exclude the connection that returned the empty turn", async () => {
  const { calls, select } = recordingSelector({ "conn-a": { connectionId: "conn-b" } });
  const next = await pickEmptyTurnRetryCredentials(select, {
    ...unconstrained,
    provider: "gemini",
    model: "gemini-2.5-flash",
    current: { connectionId: "conn-a" },
  });
  assert.deepEqual(next, { connectionId: "conn-b" });
  assert.deepEqual(calls, [{ exclude: "conn-a", allowed: null }]);
});

test("retry credentials fall back to the normal selection when nothing else is eligible", async () => {
  const { calls, select } = recordingSelector({
    "conn-a": { allRateLimited: true, retryAfter: "later" },
    null: { connectionId: "conn-a" },
  });
  const next = await pickEmptyTurnRetryCredentials(select, {
    ...unconstrained,
    provider: "gemini",
    model: "gemini-2.5-flash",
    current: { connectionId: "conn-a" },
  });
  assert.deepEqual(next, { connectionId: "conn-a" }, "a single slot replays itself");
  assert.deepEqual(calls, [
    { exclude: "conn-a", allowed: null },
    { exclude: null, allowed: null },
  ]);
});

test("both retry selections stay inside the key's connection allowlist", async () => {
  const { calls, select } = recordingSelector({
    "conn-a": { blockedByKeyPolicy: true, blockedCount: 1 },
    null: { connectionId: "conn-a" },
  });
  const next = await pickEmptyTurnRetryCredentials(select, {
    ...unconstrained,
    apiKey: { allowedConnections: ["conn-a", 7, " "] },
    provider: "gemini",
    model: null,
    current: { connectionId: "conn-a" },
  });
  assert.deepEqual(next, { connectionId: "conn-a" });
  assert.deepEqual(calls, [
    { exclude: "conn-a", allowed: ["conn-a"] },
    { exclude: null, allowed: ["conn-a"] },
  ]);
});

test("retry credentials are null when no selection yields a connection", async () => {
  const { calls, select } = recordingSelector({});
  const next = await pickEmptyTurnRetryCredentials(select, {
    ...unconstrained,
    provider: "gemini",
    model: null,
    current: {},
  });
  assert.equal(next, null);
  assert.deepEqual(calls, [{ exclude: null, allowed: null }], "no current connection: one pick");
  const failing = async () => {
    throw new Error("selection failed");
  };
  const afterThrow = await pickEmptyTurnRetryCredentials(failing, {
    ...unconstrained,
    provider: "gemini",
    model: null,
    current: { connectionId: "conn-a" },
  });
  assert.equal(afterThrow, null);
});

test("a lease, a pinned connection or a quota key replays without a selection", async () => {
  const cases = [
    { ...unconstrained, leased: true },
    { ...unconstrained, forcedConnectionId: "conn-a" },
    { ...unconstrained, apiKey: { allowedQuotas: ["pool-1"] } },
  ];
  for (const routing of cases) {
    const { calls, select } = recordingSelector({ "conn-a": { connectionId: "conn-b" } });
    const current = { connectionId: "conn-a" };
    const next = await pickEmptyTurnRetryCredentials(select, {
      ...routing,
      provider: "gemini",
      model: null,
      current,
    });
    assert.equal(next, current, JSON.stringify(routing));
    assert.deepEqual(calls, [], JSON.stringify(routing));
  }
});

test("the credential swap undoes overwritten and added fields in place", () => {
  const target: Record<string, unknown> = { connectionId: "conn-a", apiKey: "key-a" };
  const restore = swapCredentialsInPlace(target, {
    connectionId: "conn-b",
    apiKey: "key-b",
    projectId: "project-b",
  });
  assert.deepEqual(target, { connectionId: "conn-b", apiKey: "key-b", projectId: "project-b" });
  restore();
  assert.deepEqual(target, { connectionId: "conn-a", apiKey: "key-a" });
});

test("swapping credentials onto themselves is a no-op", () => {
  const target: Record<string, unknown> = { connectionId: "conn-a" };
  const restore = swapCredentialsInPlace(target, target);
  target.accessToken = "refreshed";
  restore();
  assert.deepEqual(target, { connectionId: "conn-a", accessToken: "refreshed" });
});