Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
// Hook-level tests for the flush empty-turn retry (translate-path streams).
//
// A Gemini upstream speaks a non-OpenAI format, so the OpenAI-speaking client
// path always goes through the translate transform — the hook's
// `isTranslatePath` gate is armed. Two Gemini accounts isolate rotation:
// the first serves an empty turn, the second serves content.
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-flush-empty-retry-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.REQUIRE_API_KEY = "false";
process.env.DASHBOARD_PASSWORD = "";
process.env.INITIAL_PASSWORD = "";
delete process.env.JWT_SECRET;
if (!process.env.API_KEY_SECRET) {
process.env.API_KEY_SECRET = `test-flush-empty-retry-${Date.now()}`;
}
process.env.STREAM_READINESS_TIMEOUT_MS = "1000";
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
const leaseDb = await import("../../src/lib/db/exclusiveConnectionLeases.ts");
const callLogs = await import("../../src/lib/usage/callLogs.ts");
const { handleChat } = await import("../../src/sse/handlers/chat.ts");
const { initTranslators } = await import("../../open-sse/translator/index.ts");
const { clearInflight } = await import("../../open-sse/services/requestDedup.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
const originalFetch = globalThis.fetch;
const FLAG = "FLUSH_EMPTY_RETRY_ENABLED";
const ORIGINAL_FLAG = process.env[FLAG];
function setFlag(enabled: boolean) {
if (enabled) process.env[FLAG] = "true";
else delete process.env[FLAG];
}
async function resetStorage() {
clearInflight();
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
resetAllCircuitBreakers();
initTranslators();
}
async function seedGemini(name: string, apiKey: string) {
const row = (await providersDb.createProviderConnection({
provider: "gemini",
authType: "apikey",
name,
apiKey,
isActive: true,
testStatus: "active",
})) as { id: string };
return { id: row.id, apiKey };
}
// Anonymized replay of the 19/09 reasoning-only payload shape: 74 tokens of
// thinking text (native thought part), empty message text, terminal stop.
function reasoningOnlyStreamResponse(): Response {
const thinking = "r".repeat(74);
return new Response(
`data: ${JSON.stringify({
candidates: [{ content: { parts: [{ text: thinking, thought: true }] } }],
})}\n\ndata: ${JSON.stringify({
candidates: [{ finishReason: "STOP" }],
})}\n\n`,
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
);
}
// Anonymized replay of the 19/09 zero-chunk payload shape: no candidates at
// all (the upstream turn emitted nothing translatable).
function zeroChunkStreamResponse(): Response {
return new Response(
`data: ${JSON.stringify({ candidates: [] })}\n\n` +
`data: ${JSON.stringify({ candidates: [] })}\n\n`,
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
);
}
// The upstream sent a first frame with nothing usable in it, then the
// connection dropped (undici surfaces this as a TypeError "terminated" on the
// body). The first frame lets the stream readiness gate pass, so the failure
// happens while the flush hook is buffering the turn.
function droppedStreamResponse(): Response {
let sent = false;
const stream = new ReadableStream<Uint8Array>({
pull(controller) {
if (!sent) {
sent = true;
controller.enqueue(
new TextEncoder().encode(`data: ${JSON.stringify({ candidates: [] })}\n\n`)
);
return;
}
controller.error(new TypeError("terminated"));
},
});
return new Response(stream, { status: 200, headers: { "Content-Type": "text/event-stream" } });
}
function contentStreamResponse(text: string): Response {
return new Response(
`data: ${JSON.stringify({
candidates: [{ content: { parts: [{ text }] } }],
})}\n\ndata: ${JSON.stringify({
candidates: [{ finishReason: "STOP" }],
})}\n\ndata: [DONE]\n\n`,
{ status: 200, headers: { "Content-Type": "text/event-stream" } }
);
}
function streamRequest(extraHeaders: Record<string, string> = {}) {
const nonce = `flush-empty-retry-${Date.now()}-${Math.random().toString(36).slice(2)}`;
return new Request("http://localhost/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
...extraHeaders,
},
body: JSON.stringify({
model: "gemini/gemini-2.5-flash",
messages: [{ role: "user", content: `Reply with OK only. ${nonce}` }],
max_tokens: 64,
stream: true,
temperature: 0,
}),
});
}
function stubFetch(dispatches: string[], handler: (auth: string, callIndex: number) => Response) {
globalThis.fetch = (async (_url: unknown, init: { headers?: unknown }) => {
const headers = new Headers((init?.headers ?? {}) as HeadersInit);
const auth = headers.get("authorization") ?? headers.get("x-goog-api-key") ?? "";
const callIndex = dispatches.length;
dispatches.push(auth);
return handler(auth, callIndex);
}) as typeof fetch;
}
async function drainText(response: Response): Promise<string> {
const text = await response.text().catch(() => "");
return text;
}
test.beforeEach(async () => {
globalThis.fetch = originalFetch;
setFlag(true);
await resetStorage();
});
test.afterEach(async () => {
await new Promise((r) => setTimeout(r, 50));
globalThis.fetch = originalFetch;
if (ORIGINAL_FLAG === undefined) delete process.env[FLAG];
else process.env[FLAG] = ORIGINAL_FLAG;
await resetStorage();
});
test.after(async () => {
globalThis.fetch = originalFetch;
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("flag off: empty reasoning-only turn exposes current behavior with zero retry", async () => {
setFlag(false);
await seedGemini("gemini-off-a", "sk-flush-off-a");
await seedGemini("gemini-off-b", "sk-flush-off-b");
const dispatches: string[] = [];
stubFetch(dispatches, () => reasoningOnlyStreamResponse());
const response = await handleChat(streamRequest());
await drainText(response);
assert.equal(
dispatches.length,
1,
`flag off must issue exactly 1 dispatch, got ${dispatches.length}`
);
});
test("reasoning-only turn retries once through the normal credential path and serves content", async () => {
await seedGemini("gemini-empty-a", "sk-flush-empty-a");
await seedGemini("gemini-content-b", "sk-flush-content-b");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(streamRequest());
const bodyText = await drainText(response);
assert.equal(dispatches.length, 2, `expected initial + 1 retry, got ${dispatches.length}`);
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});
test("persistent empty turns exhaust the budget with exactly 5 dispatches", async () => {
await seedGemini("gemini-empty2-a", "sk-flush-empty2-a");
await seedGemini("gemini-empty2-b", "sk-flush-empty2-b");
const dispatches: string[] = [];
stubFetch(dispatches, () => zeroChunkStreamResponse());
const response = await handleChat(streamRequest());
await drainText(response);
assert.equal(
dispatches.length,
5,
`budget 4 means 1 initial + 4 retries, got ${dispatches.length}`
);
});
test("single slot retries the same account and serves content", async () => {
await seedGemini("gemini-lone", "sk-flush-lone");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(streamRequest());
const bodyText = await drainText(response);
assert.equal(
dispatches.length,
2,
`single slot replays the same account, got ${dispatches.length}`
);
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});
test("a stream that drops before anything reaches the client retries and serves content", async () => {
await seedGemini("gemini-drop-a", "sk-flush-drop-a");
await seedGemini("gemini-drop-b", "sk-flush-drop-b");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? droppedStreamResponse() : contentStreamResponse("served-after-drop")
);
const response = await handleChat(streamRequest());
const bodyText = await drainText(response);
assert.equal(dispatches.length, 2, `expected initial + 1 retry, got ${dispatches.length}`);
assert.match(bodyText, /served-after-drop/, "client must receive the retry content");
});
test("persistent stream drops exhaust the same budget as empty turns", async () => {
await seedGemini("gemini-drop2-a", "sk-flush-drop2-a");
await seedGemini("gemini-drop2-b", "sk-flush-drop2-b");
const dispatches: string[] = [];
stubFetch(dispatches, () => droppedStreamResponse());
const response = await handleChat(streamRequest());
await drainText(response);
assert.equal(
dispatches.length,
5,
`budget 4 means 1 initial + 4 retries, got ${dispatches.length}`
);
});
test("flag off: a dropped stream is not retried", async () => {
setFlag(false);
await seedGemini("gemini-drop3-a", "sk-flush-drop3-a");
await seedGemini("gemini-drop3-b", "sk-flush-drop3-b");
const dispatches: string[] = [];
stubFetch(dispatches, () => droppedStreamResponse());
const response = await handleChat(streamRequest());
await drainText(response);
assert.equal(
dispatches.length,
1,
`flag off must issue exactly 1 dispatch, got ${dispatches.length}`
);
});
test("the retry moves off the connection that returned the empty turn", async () => {
await seedGemini("gemini-move-a", "sk-flush-move-a");
await seedGemini("gemini-move-b", "sk-flush-move-b");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(streamRequest());
const bodyText = await drainText(response);
assert.equal(dispatches.length, 2, `expected initial + 1 retry, got ${dispatches.length}`);
assert.notEqual(dispatches[1], dispatches[0], "the retry must not replay the empty connection");
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});
test("a failed retry logs the call against the connection that served it", async () => {
// Round-robin with a sticky limit of 1 rotates on every pick, so the retry
// lands on the other account even without an explicit exclusion.
await settingsDb.updateSettings({ fallbackStrategy: "round-robin", stickyRoundRobinLimit: 1 });
const served = await seedGemini("gemini-kept-a", "sk-flush-kept-a");
const other = await seedGemini("gemini-kept-b", "sk-flush-kept-b");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0
? reasoningOnlyStreamResponse()
: new Response(JSON.stringify({ error: { message: "upstream down" } }), { status: 500 })
);
const response = await handleChat(streamRequest());
await drainText(response);
assert.deepEqual(dispatches, [served.apiKey, other.apiKey]);
await callLogs.waitForCallLogSaves(2000);
const logged = (await callLogs.getCallLogs({})) as Array<{ connectionId?: string | null }>;
assert.equal(logged.length, 1, `expected one call log, got ${logged.length}`);
assert.equal(
logged[0].connectionId,
served.id,
"the kept response came from the first account, not the failed retry"
);
});
test("the retry stays inside the key's connection allowlist", async () => {
// Seeded first, so fill-first would pick it for an unrestricted selection.
await seedGemini("gemini-outside", "sk-flush-outside");
const first = await seedGemini("gemini-allowed-a", "sk-flush-allowed-a");
const second = await seedGemini("gemini-allowed-b", "sk-flush-allowed-b");
const key = await apiKeysDb.createApiKey("allowlisted-flush", "test", [], {
allowedConnections: [first.id, second.id],
});
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(streamRequest({ Authorization: `Bearer ${key.key}` }));
const bodyText = await drainText(response);
assert.deepEqual(dispatches, [first.apiKey, second.apiKey]);
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});
test("a pinned connection replays itself instead of rotating", async () => {
await seedGemini("gemini-pin-a", "sk-flush-pin-a");
const pinned = await seedGemini("gemini-pin-b", "sk-flush-pin-b");
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(streamRequest({ "x-omniroute-connection": pinned.id }));
const bodyText = await drainText(response);
assert.deepEqual(dispatches, [pinned.apiKey, pinned.apiKey]);
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});
test("a managed lease replays the leased connection instead of rotating", async () => {
const owner = `vlo_${"E".repeat(43)}`;
await seedGemini("gemini-lease-a", "sk-flush-lease-a");
const leased = await seedGemini("gemini-lease-b", "sk-flush-lease-b");
const key = await apiKeysDb.createApiKey("managed-flush", "test", ["lease:exclusive"], {
allowedConnections: [leased.id],
});
const acquired = leaseDb.acquireExclusiveConnectionLease({
leaseOwnerId: owner,
apiKeyId: key.id,
provider: "gemini",
connectionId: leased.id,
});
assert.equal(acquired.kind, "ACQUIRED");
if (acquired.kind !== "ACQUIRED") return;
const dispatches: string[] = [];
stubFetch(dispatches, (_auth, callIndex) =>
callIndex === 0 ? reasoningOnlyStreamResponse() : contentStreamResponse("served-after-retry")
);
const response = await handleChat(
streamRequest({
Authorization: `Bearer ${key.key}`,
"X-OmniRoute-Lease-Owner": owner,
"X-OmniRoute-Lease-Generation": String(acquired.lease.generation),
})
);
const bodyText = await drainText(response);
assert.deepEqual(dispatches, [leased.apiKey, leased.apiKey]);
assert.match(bodyText, /served-after-retry/, "client must receive the retry content");
});