Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(responses): lazy SSE headers — restore pre-stream failover on /v1/responses (#192)

Tashfeen48688e1aa57a
diff --git a/server/src/__tests__/routes/proxy-empty-completion.test.ts b/server/src/__tests__/routes/proxy-empty-completion.test.ts
index 8a0ac928..77411fa2 100644
--- a/server/src/__tests__/routes/proxy-empty-completion.test.ts
+++ b/server/src/__tests__/routes/proxy-empty-completion.test.ts
@@ -142,6 +142,30 @@ describe('Empty-completion failover', () => {
expect(streamChatCompletion).toHaveBeenCalledTimes(2);
});
+ it('/v1/responses (stream): a connect-time provider error fails over instead of dying mid-"stream"', async () => {
+ // Regression: headers used to be sent BEFORE the provider call, so a 503
+ // at stream open was misclassified as mid-stream → returned to the client
+ // with no failover and no cooldown (observed as 17 consecutive 503s to
+ // the same model). With lazy headers it must take the retry path.
+ async function* failsAtOpen(): AsyncGenerator<any> {
+ throw new Error('OpenRouter API error 503: Provider returned error');
+ }
+ streamChatCompletion
+ .mockReturnValueOnce(failsAtOpen())
+ .mockReturnValueOnce(goodStream());
+
+ const { status, raw } = await post(app, '/v1/responses', {
+ input: 'hi',
+ stream: true,
+ }, key);
+
+ expect(status).toBe(200);
+ expect(raw).toContain('response.completed');
+ expect(raw).toContain('streamed answer');
+ expect(raw).not.toContain('response.failed');
+ expect(streamChatCompletion).toHaveBeenCalledTimes(2);
+ });
+
it('logs the empty attempt as an error and the failover as success', async () => {
chatCompletion
.mockResolvedValueOnce(EMPTY_RESULT)
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 4a54c866..edbe7dc7 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -364,21 +364,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
try {
if (stream) {
- // Headers + response.created on the first attempt that gets this far.
- if (!streamStarted) {
- res.setHeader('Content-Type', 'text/event-stream');
- res.setHeader('Cache-Control', 'no-cache');
- res.setHeader('Connection', 'keep-alive');
- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
- const skeleton = {
- id: responseId, object: 'response', created_at: nowUnix(),
- status: 'in_progress', model: route.modelId, output: [], output_text: '',
- };
- sse('response.created', { response: skeleton });
- sse('response.in_progress', { response: skeleton });
- streamStarted = true;
- }
-
let outputIndex = 0;
let msgItemId: string | null = null;
let msgText = '';
@@ -389,6 +374,32 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);
for await (const chunk of gen) {
+ // LAZY header set — headers + the response.created/in_progress
+ // skeleton go out only once the provider actually streams a chunk.
+ // Sending them before the provider call (the previous behavior)
+ // committed the SSE response, so a provider error AT STREAM OPEN —
+ // e.g. OpenRouter 503ing a large-context request — was misclassified
+ // as mid-stream and returned to the client with NO failover and NO
+ // cooldown; the next request then hit the same broken model again
+ // (observed: 17 consecutive 503s to the same model while the rest of
+ // the chain sat idle). With lazy headers a connect-time error
+ // bubbles to the catch with streamStarted=false and takes the normal
+ // retry path. Mirrors the proxy's streaming handler.
+ if (!streamStarted) {
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.setHeader('Connection', 'keep-alive');
+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
+ const skeleton = {
+ id: responseId, object: 'response', created_at: nowUnix(),
+ status: 'in_progress', model: route.modelId, output: [], output_text: '',
+ };
+ sse('response.created', { response: skeleton });
+ sse('response.in_progress', { response: skeleton });
+ streamStarted = true;
+ }
+
const delta = chunk.choices[0]?.delta;
if (!delta) continue;