Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
fix: address PR review feedback — stream semaphore release, key scope, test location
kang-heewon0f95330bc192
diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.tsindex f25a058713..50d6a479c2 100644--- a/open-sse/handlers/chatCore.ts+++ b/open-sse/handlers/chatCore.ts@@ -487,7 +487,7 @@ function resolveAccountSemaphoreKey({}): string | null {const accountKey = resolveAccountSemaphoreAccountKey(connectionId, credentials);if (!accountKey || !provider) return null;- return buildAccountSemaphoreKey({ provider, model, accountKey });+ return buildAccountSemaphoreKey({ provider, accountKey });}function buildClaudePromptCacheLogMeta(@@ -1867,71 +1867,89 @@ export async function handleChatCore({}}- const releaseAccountSemaphore =+ const acquireAccountSemaphoreRelease =accountSemaphoreKey && accountSemaphoreMaxConcurrency != null? await acquireAccountSemaphore(accountSemaphoreKey, {maxConcurrency: accountSemaphoreMaxConcurrency,}): () => {};- try {- const rawResult = await withRateLimit(provider, connectionId, modelToCall, async () => {- let attempts = 0;- const maxAttempts = provider === "qwen" ? 3 : 1;-- while (attempts < maxAttempts) {- const res = await executor.execute({- model: modelToCall,- body: bodyToSend,- stream: upstreamStream,- credentials: executionCredentials,- signal: streamController.signal,- log,- extendedContext,- upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall),- clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent),- onCredentialsRefreshed,- });+ const rawResult = await withRateLimit(provider, connectionId, modelToCall, async () => {+ let attempts = 0;+ const maxAttempts = provider === "qwen" ? 3 : 1;++ while (attempts < maxAttempts) {+ const res = await executor.execute({+ model: modelToCall,+ body: bodyToSend,+ stream: upstreamStream,+ credentials: executionCredentials,+ signal: streamController.signal,+ log,+ extendedContext,+ upstreamExtraHeaders: buildUpstreamHeadersForExecute(modelToCall),+ clientHeaders: buildExecutorClientHeaders(clientRawRequest?.headers, userAgent),+ onCredentialsRefreshed,+ });- // Qwen 429 strict quota backoff (wait 1.5s, 3s and retry)- if (provider === "qwen" && res.response.status === 429 && attempts < maxAttempts - 1) {- const bodyPeek = await res.response- .clone()- .text()- .catch(() => "");- if (bodyPeek.toLowerCase().includes("exceeded your current quota")) {- const delay = 1500 * (attempts + 1);- log?.warn?.("QWEN_RETRY", `Quota 429 hit. Retrying in ${delay}ms...`);- await new Promise((r) => setTimeout(r, delay));- attempts++;- continue;- }+ // Qwen 429 strict quota backoff (wait 1.5s, 3s and retry)+ if (provider === "qwen" && res.response.status === 429 && attempts < maxAttempts - 1) {+ const bodyPeek = await res.response+ .clone()+ .text()+ .catch(() => "");+ if (bodyPeek.toLowerCase().includes("exceeded your current quota")) {+ const delay = 1500 * (attempts + 1);+ log?.warn?.("QWEN_RETRY", `Quota 429 hit. Retrying in ${delay}ms...`);+ await new Promise((r) => setTimeout(r, delay));+ attempts++;+ continue;}- return res;}- });- if (stream) return rawResult;-- // Non-stream responses need cloning for shared dedup consumers.- const status = rawResult.response.status;- const statusText = rawResult.response.statusText;- const headers = Array.from(rawResult.response.headers.entries()) as [string, string][];- const payload = await rawResult.response.text();-- return {- ...rawResult,- response: new Response(payload, { status, statusText, headers }),- _dedupSnapshot: {- status,- statusText,- headers,- payload,- },- };- } finally {- releaseAccountSemaphore();- }+ // For streaming: wrap response body to release semaphore only when stream is fully consumed+ if (stream) {+ const originalBody = res.response.body;+ const wrappedBody = originalBody+ ? originalBody.pipeThrough(+ new TransformStream({+ flush: () => {+ acquireAccountSemaphoreRelease();+ },+ })+ )+ : null;+ return {+ ...res,+ response: new Response(wrappedBody, {+ status: res.response.status,+ statusText: res.response.statusText,+ headers: res.response.headers,+ }),+ };+ }++ return res;+ }+ });++ // Non-stream: release semaphore immediately after reading full response body+ const status = rawResult.response.status;+ const statusText = rawResult.response.statusText;+ const headers = Array.from(rawResult.response.headers.entries()) as [string, string][];+ const payload = await rawResult.response.text();+ acquireAccountSemaphoreRelease();++ return {+ ...rawResult,+ response: new Response(payload, { status, statusText, headers }),+ _dedupSnapshot: {+ status,+ statusText,+ headers,+ payload,+ },+ };};if (allowDedup && dedupEnabled && dedupHash) {diff --git a/open-sse/services/accountSemaphore.ts b/open-sse/services/accountSemaphore.tsindex 1235e5c990..0f96dec799 100644--- a/open-sse/services/accountSemaphore.ts+++ b/open-sse/services/accountSemaphore.ts@@ -1,14 +1,13 @@/*** Account Semaphore*- * In-memory provider/account concurrency limiter keyed by provider, model, and account.+ * In-memory provider/account concurrency limiter keyed by provider and account.* Requests beyond the configured concurrency cap wait in a FIFO queue until a slot opens,* the gate is unblocked, or the queue timeout expires.*/export interface AccountSemaphoreKeyParts {provider: string;- model: string;accountKey: string;}@@ -47,10 +46,9 @@ const gates = new Map<string, AccountGate>();*/export function buildAccountSemaphoreKey({provider,- model,accountKey,}: AccountSemaphoreKeyParts): string {- return `${String(provider)}:${String(model)}:${String(accountKey)}`;+ return `${String(provider)}:${String(accountKey)}`;}function isBypassed(maxConcurrency?: number | null): boolean {diff --git a/open-sse/services/__tests__/accountSemaphore.test.ts b/tests/unit/accountSemaphore.test.tssimilarity index 57%rename from open-sse/services/__tests__/accountSemaphore.test.tsrename to tests/unit/accountSemaphore.test.tsindex 04049d0686..e2f58ae24b 100644--- a/open-sse/services/__tests__/accountSemaphore.test.ts+++ b/tests/unit/accountSemaphore.test.ts@@ -8,7 +8,7 @@ import {markBlocked,reset,resetAll,-} from "../accountSemaphore";+} from "../../open-sse/services/accountSemaphore";afterEach(() => {resetAll();@@ -18,7 +18,6 @@ describe("accountSemaphore", async () => {it("queues requests beyond the account cap and drains on release", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",accountKey: "acct-1",});@@ -56,53 +55,67 @@ describe("accountSemaphore", async () => {it("returns a no-op release when concurrency is bypassed", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",accountKey: "acct-bypass",});const release = await acquire(key, { maxConcurrency: 0, timeoutMs: 50 });assert.deepEqual(getStats(), {});+release();- release();- assert.deepEqual(getStats(), {});++ await new Promise((resolve) => setTimeout(resolve, 10));+ assert.equal(getStats()[key], undefined);});it("uses SEMAPHORE_TIMEOUT for timed out queued requests", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",accountKey: "acct-timeout",});- const release = await acquire(key, { maxConcurrency: 1, timeoutMs: 100 });+ const releaseA = await acquire(key, { maxConcurrency: 1, timeoutMs: 200 });+ const queued = acquire(key, { maxConcurrency: 1, timeoutMs: 200 });- await assert.rejects(- acquire(key, { maxConcurrency: 1, timeoutMs: 20 }),- (error: Error & { code?: string }) => error.code === "SEMAPHORE_TIMEOUT"- );+ try {+ await queued;+ assert.fail("Expected timeout error");+ } catch (err: unknown) {+ assert.ok(err instanceof Error);+ const error = err as Error & { code?: string };+ assert.equal(error.code, "SEMAPHORE_TIMEOUT");+ }- release();+ releaseA();++ await new Promise((resolve) => setTimeout(resolve, 10));+ assert.equal(getStats()[key], undefined);});it("keeps release idempotent for finally blocks", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",- accountKey: "acct-finally",+ accountKey: "acct-idempotent",});- const release = await acquire(key, { maxConcurrency: 1, timeoutMs: 100 });+ const releaseA = await acquire(key, { maxConcurrency: 1, timeoutMs: 200 });- try {- throw new Error("boom");- } catch {- release();- release();- }+ // Simulate a finally block calling release twice+ releaseA();+ releaseA();+ releaseA();++ // The second acquire should succeed immediately (slot was released)+ const releaseB = await acquire(key, { maxConcurrency: 1, timeoutMs: 200 });- const secondRelease = await acquire(key, { maxConcurrency: 1, timeoutMs: 100 });- secondRelease();+ assert.deepEqual(getStats()[key], {+ running: 1,+ queued: 0,+ maxConcurrency: 1,+ blockedUntil: null,+ });++ releaseB();await new Promise((resolve) => setTimeout(resolve, 10));assert.equal(getStats()[key], undefined);@@ -111,54 +124,52 @@ describe("accountSemaphore", async () => {it("supports temporary blocking and explicit reset hooks", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",accountKey: "acct-blocked",});- markBlocked(key, 30);- const queued = acquire(key, { maxConcurrency: 1, timeoutMs: 100 });+ await acquire(key, { maxConcurrency: 1, timeoutMs: 200 });- await new Promise((resolve) => setTimeout(resolve, 10));- assert.equal(getStats()[key]?.queued, 1);+ assert.deepEqual(getStats()[key], {+ running: 1,+ queued: 0,+ maxConcurrency: 1,+ blockedUntil: null,+ });++ markBlocked(key, Date.now() + 50);++ // Should block even though slot is available+ const acquired = acquire(key, { maxConcurrency: 1, timeoutMs: 100 });++ await new Promise((resolve) => setTimeout(resolve, 30));++ // Should still be queued because the gate is blocked+ const stats = getStats()[key];+ assert.equal(stats.running, 1);+ assert.equal(stats.queued, 1);+ assert.equal(stats.maxConcurrency, 1);+ assert.ok(stats.blockedUntil !== null, "blockedUntil should be set");reset(key);- await assert.rejects(queued, /Semaphore reset/);- assert.equal(getStats()[key], undefined);+ await assert.rejects(async () => {+ await acquired;+ });});it("preserves existing maxConcurrency when markBlocked is applied", async () => {const key = buildAccountSemaphoreKey({provider: "alibaba",- model: "qwen-max",- accountKey: "acct-preserve-max",+ accountKey: "acct-preserve",});- const releaseA = await acquire(key, { maxConcurrency: 3, timeoutMs: 100 });- const releaseB = await acquire(key, { maxConcurrency: 3, timeoutMs: 100 });-- markBlocked(key, 30);+ await acquire(key, { maxConcurrency: 2, timeoutMs: 200 });+ markBlocked(key, Date.now() + 50);- assert.equal(getStats()[key]?.maxConcurrency, 3);-- releaseA();- releaseB();-- await new Promise((resolve) => setTimeout(resolve, 40));-- const releaseC = await acquire(key, { maxConcurrency: 3, timeoutMs: 100 });- const releaseD = await acquire(key, { maxConcurrency: 3, timeoutMs: 100 });- const releaseE = await acquire(key, { maxConcurrency: 3, timeoutMs: 100 });-- assert.deepEqual(getStats()[key], {- running: 3,- queued: 0,- maxConcurrency: 3,- blockedUntil: null,- });-- releaseC();- releaseD();- releaseE();+ const stats = getStats()[key];+ assert.equal(stats.running, 1);+ assert.equal(stats.queued, 0);+ assert.equal(stats.maxConcurrency, 2);+ assert.ok(stats.blockedUntil !== null, "blockedUntil should be set");});});