7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(responses): lazy SSE headers — restore pre-stream failover on /v1/responses (#192)
Tashfeen48688e1aa57a
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.tsindex 8a0ac928..77411fa2 100644--- a/server/src/__tests__/routes/proxy-empty-completion.test.ts+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts@@ -142,6 +142,30 @@ describe('Empty-completion failover', () => {expect(streamChatCompletion).toHaveBeenCalledTimes(2);});+ it('/v1/responses (stream): a connect-time provider error fails over instead of dying mid-"stream"', async () => {+ // Regression: headers used to be sent BEFORE the provider call, so a 503+ // at stream open was misclassified as mid-stream → returned to the client+ // with no failover and no cooldown (observed as 17 consecutive 503s to+ // the same model). With lazy headers it must take the retry path.+ async function* failsAtOpen(): AsyncGenerator<any> {+ throw new Error('OpenRouter API error 503: Provider returned error');+ }+ streamChatCompletion+ .mockReturnValueOnce(failsAtOpen())+ .mockReturnValueOnce(goodStream());++ const { status, raw } = await post(app, '/v1/responses', {+ input: 'hi',+ stream: true,+ }, key);++ expect(status).toBe(200);+ expect(raw).toContain('response.completed');+ expect(raw).toContain('streamed answer');+ expect(raw).not.toContain('response.failed');+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);+ });+it('logs the empty attempt as an error and the failover as success', async () => {chatCompletion.mockResolvedValueOnce(EMPTY_RESULT)diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 4a54c866..edbe7dc7 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -364,21 +364,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {try {if (stream) {- // Headers + response.created on the first attempt that gets this far.- if (!streamStarted) {- res.setHeader('Content-Type', 'text/event-stream');- res.setHeader('Cache-Control', 'no-cache');- res.setHeader('Connection', 'keep-alive');- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);- const skeleton = {- id: responseId, object: 'response', created_at: nowUnix(),- status: 'in_progress', model: route.modelId, output: [], output_text: '',- };- sse('response.created', { response: skeleton });- sse('response.in_progress', { response: skeleton });- streamStarted = true;- }-let outputIndex = 0;let msgItemId: string | null = null;let msgText = '';@@ -389,6 +374,32 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);for await (const chunk of gen) {+ // LAZY header set — headers + the response.created/in_progress+ // skeleton go out only once the provider actually streams a chunk.+ // Sending them before the provider call (the previous behavior)+ // committed the SSE response, so a provider error AT STREAM OPEN —+ // e.g. OpenRouter 503ing a large-context request — was misclassified+ // as mid-stream and returned to the client with NO failover and NO+ // cooldown; the next request then hit the same broken model again+ // (observed: 17 consecutive 503s to the same model while the rest of+ // the chain sat idle). With lazy headers a connect-time error+ // bubbles to the catch with streamStarted=false and takes the normal+ // retry path. Mirrors the proxy's streaming handler.+ if (!streamStarted) {+ res.setHeader('Content-Type', 'text/event-stream');+ res.setHeader('Cache-Control', 'no-cache');+ res.setHeader('Connection', 'keep-alive');+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));+ const skeleton = {+ id: responseId, object: 'response', created_at: nowUnix(),+ status: 'in_progress', model: route.modelId, output: [], output_text: '',+ };+ sse('response.created', { response: skeleton });+ sse('response.in_progress', { response: skeleton });+ streamStarted = true;+ }+const delta = chunk.choices[0]?.delta;if (!delta) continue;