7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): restore session reasoning_content dropped by clients on replay (#797) (#805)
suantea (asuan_suan)93d08a4dd581
diff --git a/server/src/__tests__/routes/proxy-tools.test.ts b/server/src/__tests__/routes/proxy-tools.test.tsindex 85209aef..d9807e97 100644--- a/server/src/__tests__/routes/proxy-tools.test.ts+++ b/server/src/__tests__/routes/proxy-tools.test.ts@@ -220,4 +220,155 @@ describe('Proxy tool-calling support', () => {expect(providerBody.messages[1].role).toBe('assistant');expect(providerBody.messages[1].reasoning_content).toBe('Let me reason about this step by step...');});++ it('restores session reasoning_content the client dropped on replay (#797)', async () => {+ const origFetch = global.fetch;+ let turn = 0;+ let providerBody: any = null;++ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ turn += 1;+ if (turn === 2) providerBody = JSON.parse((init as any).body);+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-r', object: 'chat.completion', created: 1, model: 'm',+ choices: [{+ index: 0,+ message: {+ role: 'assistant',+ content: turn === 1 ? 'first answer' : 'second answer',+ // Turn 1 is a thinking turn: the provider returns a trace the+ // proxy must remember for the session.+ ...(turn === 1 ? { reasoning_content: 'session trace from turn one' } : {}),+ },+ finish_reason: 'stop',+ }],+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const sessHeaders = { ...authHeaders(), 'x-session-id': 'sess-797' };++ // Turn 1: thinking model returns reasoning_content → proxy records it.+ const first = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'think then answer' }],+ }, sessHeaders);+ expect(first.status).toBe(200);++ // Turn 2: same session; opencode-style replay strips reasoning_content+ // (AI-SDK convertToOpenAICompatibleChatMessages). The proxy must restore+ // the trace it returned last turn or OpenCode Zen 400s.+ const second = await request(app, 'POST', '/v1/chat/completions', {+ messages: [+ { role: 'user', content: 'think then answer' },+ { role: 'assistant', content: 'first answer' },+ { role: 'user', content: 'continue' },+ ],+ }, sessHeaders);+ expect(second.status).toBe(200);+ expect(providerBody.messages[1].role).toBe('assistant');+ expect(providerBody.messages[1].reasoning_content).toBe('session trace from turn one');+ });++ it('sends the client\'s messages untouched when the session remembers no reasoning (#797)', async () => {+ const origFetch = global.fetch;+ let providerBody: any = null;++ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ providerBody = JSON.parse((init as any).body);+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-p', object: 'chat.completion', created: 1, model: 'm',+ // No reasoning_content anywhere in this session.+ choices: [{ index: 0, message: { role: 'assistant', content: 'plain answer 2' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const sent = [+ { role: 'user', content: 'no thinking in this session' },+ { role: 'assistant', content: 'plain answer' },+ { role: 'user', content: 'continue' },+ ];++ const { status } = await request(app, 'POST', '/v1/chat/completions', { messages: sent },+ { ...authHeaders(), 'x-session-id': 'sess-797-plain' });++ expect(status).toBe(200);+ // Byte-identical: no restore, no empty-string filler, no reordering.+ expect(providerBody.messages).toEqual(sent);+ expect(providerBody.messages.some((m: any) => 'reasoning_content' in m)).toBe(false);+ });++ it('restores only into the outbound copy, so a failover hop still sends the client\'s bytes (#797)', async () => {+ const origFetch = global.fetch;+ const calls: any[] = [];++ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ calls.push(JSON.parse((init as any).body));+ // Call 1 is turn 1 (records the trace). Call 2 is turn 2's first+ // attempt — rate-limited so the request fails over to another model.+ if (calls.length === 2) {+ return new Response(JSON.stringify({ error: { message: 'rate limited' } }), { status: 429 }) as any;+ }+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-f', object: 'chat.completion', created: 1, model: 'm',+ choices: [{+ index: 0,+ message: {+ role: 'assistant',+ content: 'answer',+ ...(calls.length === 1 ? { reasoning_content: 'trace bound to the first model' } : {}),+ },+ finish_reason: 'stop',+ }],+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const sessHeaders = { ...authHeaders(), 'x-session-id': 'sess-797-failover' };++ const first = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'think then answer' }],+ }, sessHeaders);+ expect(first.status).toBe(200);++ const second = await request(app, 'POST', '/v1/chat/completions', {+ messages: [+ { role: 'user', content: 'think then answer' },+ { role: 'assistant', content: 'answer' },+ { role: 'user', content: 'continue' },+ ],+ }, sessHeaders);+ expect(second.status).toBe(200);+ expect(calls).toHaveLength(3);++ // Attempt 1 goes to the model that produced the trace — restored.+ expect(calls[1].messages[1].reasoning_content).toBe('trace bound to the first model');+ // Attempt 2 is a different model. The restore lived on the outbound copy+ // only, so this hop carries exactly what the client sent — no leaked+ // reasoning_content from the mutated request body.+ expect(calls[2].model).not.toBe(calls[1].model);+ expect(calls[2].messages[1].reasoning_content).toBeUndefined();+ expect(calls[2].messages.some((m: any) => 'reasoning_content' in m)).toBe(false);+ });});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex eb005ce1..8ea7025a 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -148,6 +148,85 @@ export { exhaustedRetryError };const stickySessionMap = new Map<string, { modelDbId: number; lastUsed: number }>();const STICKY_TTL_MS = 30 * 60 * 1000; // 30 min session TTL+// #797: per-session memory of the last assistant turn's thinking trace.+// DeepSeek thinking models on OpenCode Zen 400 on a follow-up turn unless the+// prior `reasoning_content` is replayed; opencode (and other AI-SDK clients)+// strip the field when re-serializing history, so the proxy restores what it+// itself returned last turn. Non-thinking sessions never record an entry.+// The trace is stored WITH the model key that produced it: a remembered trace+// is only ever replayed to that same platform+model, so a session that fails+// over (or auto-routes elsewhere on the next turn) never carries one model's+// thinking into another provider's payload.+const reasoningMemory = new Map<string, { reasoning: string; modelKey: string; lastUsed: number }>();+const REASONING_TTL_MS = 30 * 60 * 1000; // 30 min, matching sticky sessions++// Platforms that reject an assistant turn WITHOUT `reasoning_content` once the+// conversation is in thinking mode, i.e. where the field has to be present on+// every assistant message and older turns need an empty-string filler. Only+// OpenCode Zen is on record for this (the DeepSeek thinking semantics behind+// #255/#797); everywhere else only the turn we actually have a trace for is+// touched, so no other provider's bytes change.+const PLATFORMS_REQUIRING_REASONING_ECHO = new Set(['opencode']);++function rememberReasoning(sessionKey: string | undefined, modelKey: string, reasoning: string) {+ if (!sessionKey || !reasoning) return;+ reasoningMemory.set(sessionKey, { reasoning, modelKey, lastUsed: Date.now() });+ if (reasoningMemory.size > 500) {+ const now = Date.now();+ for (const [key, entry] of reasoningMemory) {+ if (now - entry.lastUsed > REASONING_TTL_MS) reasoningMemory.delete(key);+ }++ // Hard cap: traces are far bigger than a sticky entry, so an all-fresh map+ // must not grow without bound. Evict oldest by lastUsed, as setStickyModel does.+ if (reasoningMemory.size > 1000) {+ const entries = [...reasoningMemory.entries()].sort((a, b) => a[1].lastUsed - b[1].lastUsed);+ const toEvict = reasoningMemory.size - 1000;+ for (let i = 0; i < toEvict; i++) reasoningMemory.delete(entries[i][0]);+ }+ }+}++// The remembered trace for this session, or undefined when there is none, it+// expired, or it came from a different model than the one about to be called.+// An expired entry is dropped on read rather than left for the size sweep.+function rememberedReasoningFor(sessionKey: string, modelKey: string): string | undefined {+ if (!sessionKey) return undefined;+ const entry = reasoningMemory.get(sessionKey);+ if (!entry) return undefined;+ if (Date.now() - entry.lastUsed > REASONING_TTL_MS) {+ reasoningMemory.delete(sessionKey);+ return undefined;+ }+ return entry.modelKey === modelKey ? entry.reasoning : undefined;+}++// Put the remembered trace back on the newest assistant turn that lost it.+// Returns a NEW array (with new objects for the messages it changes) whenever+// it changes anything — the caller's `messages` are what handoff recording,+// logging, compression and the response cache already saw, and must not move+// under them. Returns the input untouched when there is nothing to restore.+function restoreSessionReasoning(messages: ChatMessage[], reasoning: string, platform: string): ChatMessage[] {+ // Older assistant turns get "" — DeepSeek requires the field on every+ // assistant message, and an empty string satisfies it (see opencode issue+ // #24104). Only for platforms that actually enforce that; elsewhere just the+ // one turn we have a real trace for is touched.+ const fillOlderTurns = PLATFORMS_REQUIRING_REASONING_ECHO.has(platform);+ let restored: ChatMessage[] | undefined;+ let restoredLatest = false;+ for (let i = messages.length - 1; i >= 0; i -= 1) {+ const m = messages[i];+ if (m.role !== 'assistant') continue;+ // The client kept the field — nothing was dropped, leave it alone.+ if (typeof m.reasoning_content === 'string' && m.reasoning_content.length > 0) continue;+ restored ??= [...messages];+ restored[i] = { ...m, reasoning_content: restoredLatest ? '' : reasoning };+ if (!restoredLatest && !fillOlderTurns) break;+ restoredLatest = true;+ }+ return restored ?? messages;+}+function getSessionKey(messages: ChatMessage[], sessionIdHeader?: string, strategyKey?: string): string {if (sessionIdHeader) {return strategyKey ? `hdr:${sessionIdHeader}::${strategyKey}` : `hdr:${sessionIdHeader}`;@@ -1521,6 +1600,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (handoffMode !== 'off' && sessionKey) {recordIncomingMessages(sessionKey, messages);}++ // #797: key for the per-session thinking-trace memory. Read and written+ // inside the dispatch loop, where the routed platform/model is known — the+ // restore only ever touches the OUTBOUND copy, never `messages`, so handoff+ // recording above, request logging, compression and the response cache all+ // keep seeing exactly what the client sent.+ //+ // Header-less clients are covered: without x-session-id, getSessionKey hashes+ // the FIRST user message, which does not change as the conversation grows —+ // the same stability sticky sessions already rely on. Its known weakness is+ // shared here: two conversations opening with identical text share a key, so+ // the same-model + field-actually-missing gates below are what keep a+ // mis-keyed restore from reaching a payload it does not belong in.+ const reasoningSessionKey = getSessionKey(messages, sessionIdHeader, strategyKey);// A handoff can only fire when a prior model is on record for this session.// Check after recordIncomingMessages, which clears the prior model on a// fresh conversation. Stable across the retry loop (the prior model only@@ -1660,6 +1753,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {requestedModel: attempt === 0 ? requestedModelLabel : undefined,});let outboundMessages = messages;+ // #797: thinking trace accumulated from this turn's streamed deltas, then+ // remembered per-session so a follow-up whose client stripped the field+ // can have it restored (see restore block above).+ let streamReasoning = '';// Extra input tokens the injected handoff adds on this turn (0 when not// injected). Folded into the streaming success accounting, where token// counts are estimated; the non-stream path uses the provider's usage,@@ -1672,6 +1769,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {injectedHandoffTokens = handoff.injectedTokens;}+ // #797: restore the thinking trace this proxy emitted last turn, for THIS+ // model, when the client dropped it on replay. Scoped to the outbound copy+ // and to the model that produced the trace, so a failover hop and every+ // provider that never needed the field send the client's bytes unchanged.+ const rememberedReasoning = rememberedReasoningFor(reasoningSessionKey, modelKey);+ if (rememberedReasoning) {+ outboundMessages = restoreSessionReasoning(outboundMessages, rememberedReasoning, route.platform);+ }+if (stream) {// — Stream turn-integrity (#231 audit) —// The old loop forwarded upstream chunks verbatim and called any@@ -1786,6 +1892,14 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (choice.finish_reason) upstreamFinish = choice.finish_reason;+ // #797: accumulate this turn's thinking trace (native+ // reasoning_content deltas; the <think> extractor in base.ts has+ // already normalized inline tags into the same field) so a+ // follow-up request whose client stripped it can have it restored+ // from session memory.+ const reasoningDelta = choice.delta?.reasoning_content;+ if (typeof reasoningDelta === 'string' && reasoningDelta.length > 0) streamReasoning += reasoningDelta;+// Buffer tool_call deltas — emitted complete + repaired at end.for (const tc of choice.delta?.tool_calls ?? []) {const idx = tc.index ?? 0;@@ -1915,6 +2029,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {recordUpstreamSuccess(route, estimatedInputTokens + injectedHandoffTokens + totalOutputTokens);setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });+ // #797: remember this turn's thinking trace so the next request from+ // the same session can restore it (clients strip it on replay).+ if (streamReasoning.length > 0) rememberReasoning(reasoningSessionKey, modelKey, streamReasoning);traceRouteEvent('Proxy', {event: 'ok',requestId: requestGroupId,@@ -2061,6 +2178,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {?? Math.ceil((contentToString(respMsg?.content ?? '').length + respToolArgChars) / 4);const totalTokens = result.usage?.total_tokens ?? (promptTokens + completionTokens);recordUpstreamSuccess(route, totalTokens);+ // #797: remember this turn's thinking trace so the next request from+ // the same session can restore it (clients strip it on replay).+ // normalizeChoices keeps reasoning_content on the message even when it+ // folds the trace into an otherwise-empty content field.+ if (typeof respMsg?.reasoning_content === 'string' && respMsg.reasoning_content.length > 0) {+ rememberReasoning(reasoningSessionKey, modelKey, respMsg.reasoning_content);+ }// Use stickyStrategyKey (not the global strategyKey) so a group-pinned// request writes its sticky entry under the SAME key the next turn reads// from (set to the requested model id at the top of the loop). Matches the