Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(llm): surface the stream duration cap as a terminal error code
Alfredo Lopezcba325fa0079
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.tsindex f3c9b0145..aed7bc498 100644--- a/codex-llm-proxy/src/codexTransport.ts+++ b/codex-llm-proxy/src/codexTransport.ts@@ -49,6 +49,23 @@ export class CodexTransportError extends Error {}}+/**+ * The proxy cut the upstream stream on one of its two bounds. The wire code+ * stays that of CodexTransportError (`provider_unavailable` for idle silence,+ * `stream_duration_exceeded` for the total cap); `kind` only labels the metric.+ */+export class UpstreamTimeoutError extends CodexTransportError {+ constructor(+ readonly kind: 'idle' | 'total',+ code: string,+ message: string,+ details: Readonly<Record<string, number | string>>+ ) {+ super(code, message, details)+ this.name = 'UpstreamTimeoutError'+ }+}+export type StreamCodexCompletionInput = {executionTicket: stringrequestHash: string@@ -235,7 +252,8 @@ class UpstreamDeadline {) {this.total = setTimeout(() => {this.controller.abort(- new CodexTransportError(+ new UpstreamTimeoutError(+ 'total','stream_duration_exceeded','upstream stream exceeded maxStreamDurationMs',{ limitMs: this.totalMs, elapsedMs: Date.now() - this.started }@@ -257,7 +275,7 @@ class UpstreamDeadline {signal.throwIfAborted()this.idle = setTimeout(() => {this.controller.abort(- new CodexTransportError('provider_unavailable', 'upstream stream idle timeout', {+ new UpstreamTimeoutError('idle', 'provider_unavailable', 'upstream stream idle timeout', {idleTimeoutMs: this.idleMs,elapsedMs: Date.now() - this.started,})diff --git a/codex-llm-proxy/src/config.ts b/codex-llm-proxy/src/config.tsindex 2baa3094d..610850fb3 100644--- a/codex-llm-proxy/src/config.ts+++ b/codex-llm-proxy/src/config.ts@@ -1,3 +1,5 @@+import { STREAM_LIMITS } from './requestLimits.js'+export type CodexLlmProxyConfig = {runtimePort: numberadminPort: number@@ -5,6 +7,8 @@ export type CodexLlmProxyConfig = {maxBodyBytes: numbermaxStreamDurationMs: numbermaxDeadlineMs: number+ /** Lowers STREAM_LIMITS.upstreamIdleTimeoutMs; the transport never raises it. */+ upstreamIdleTimeoutMs: numberjwtIssuer: stringjwtPublicKey: stringexecutionEnabled: boolean@@ -75,6 +79,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyCenv.CODEX_LLM_PROXY_MAX_DEADLINE_MS,300_000),+ upstreamIdleTimeoutMs: requiredPositiveInt(+ 'CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS',+ env.CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,+ STREAM_LIMITS.upstreamIdleTimeoutMs+ ),jwtIssuer: env.CODEX_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',jwtPublicKey: requiredPem('CODEX_LLM_PROXY_JWT_PUBLIC_KEY', env.CODEX_LLM_PROXY_JWT_PUBLIC_KEY),executionEnabled: env.CODEX_LLM_PROXY_EXECUTION_ENABLED === 'true',diff --git a/codex-llm-proxy/src/metrics.ts b/codex-llm-proxy/src/metrics.tsindex 4c71f426f..a92fab9c5 100644--- a/codex-llm-proxy/src/metrics.ts+++ b/codex-llm-proxy/src/metrics.ts@@ -19,6 +19,12 @@ export function createProxyMetrics(register: Registry) {labelNames: ['code'],registers: [register],})+ const upstreamTimeouts = new Counter({+ name: 'codex_proxy_upstream_timeouts_total',+ help: 'Codex upstream streams cut by the proxy, by bound (idle silence or total duration)',+ labelNames: ['kind'],+ registers: [register],+ })return {observeAttempt(outcome: string, operation: string) {outcomes.inc({ outcome, operation })@@ -26,6 +32,9 @@ export function createProxyMetrics(register: Registry) {observeAttemptFailure(code: string) {attemptFailures.inc({ code })},+ observeUpstreamTimeout(kind: 'idle' | 'total') {+ upstreamTimeouts.inc({ kind })+ },observeStream(durationMs: number) {streamSeconds.observe(durationMs / 1000)},diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.tsindex 7b48aa2ac..01047c747 100644--- a/codex-llm-proxy/src/server.ts+++ b/codex-llm-proxy/src/server.ts@@ -11,6 +11,7 @@ import {listCodexModels,streamCodexCompletion,testCodexConnection,+ UpstreamTimeoutError,} from './codexTransport.js'import type { CodexLlmProxyConfig } from './config.js'import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'@@ -184,6 +185,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeDrequest: parsed.data.request,deadlineMs: parsed.data.deadlineMs,maxDeadlineMs: Math.min(config.maxDeadlineMs, config.maxStreamDurationMs),+ upstreamIdleTimeoutMs: config.upstreamIdleTimeoutMs,ticket: {jti: ticket.jti,hostRef: ticket.hostRef,@@ -224,6 +226,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeDconst mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')metrics.observeAttemptFailure(failureLabel(mapped.code))+ if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'logger.warn({@@ -379,6 +382,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {disabled: 404,ticket_replayed: 409,tool_call_limit_exceeded: 422,+ // The attempt ran for its whole stream budget. Retrying the same request+ // would spend the same budget again, so it is a gateway timeout, not 503.+ stream_duration_exceeded: 504,connection_unavailable: 503,provider_unavailable: 503,sse_buffer_exceeded: 503,diff --git a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.tsindex 97cf9add4..868bf049f 100644--- a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts+++ b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts@@ -82,6 +82,7 @@ function config(): CodexLlmProxyConfig {maxBodyBytes: 1_048_576,maxStreamDurationMs: 30_000,maxDeadlineMs: 30_000,+ upstreamIdleTimeoutMs: 300_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.tsindex 9f0d1d663..4db724354 100644--- a/codex-llm-proxy/test/server.security.test.ts+++ b/codex-llm-proxy/test/server.security.test.ts@@ -33,6 +33,7 @@ function config(overrides: Partial<CodexLlmProxyConfig> = {}): CodexLlmProxyConfmaxBodyBytes: 1024,maxStreamDurationMs: 300_000,maxDeadlineMs: 300_000,+ upstreamIdleTimeoutMs: 300_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,@@ -292,6 +293,29 @@ describe('codex-llm-proxy security surface', () => {CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS: String(Number.MAX_SAFE_INTEGER),})).toThrow(/bounded positive integer/)+ expect(() =>+ loadConfig({+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '0',+ })+ ).toThrow(/CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS must be a finite integer greater than zero/)+ })++ it('defaults the upstream idle timeout to the published STREAM_LIMITS value', () => {+ const loaded = loadConfig({+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ CODEX_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',+ CODEX_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',+ })+ expect(loaded.upstreamIdleTimeoutMs).toBe(300_000)+ expect(+ loadConfig({+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ CODEX_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',+ CODEX_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',+ CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '45000',+ }).upstreamIdleTimeoutMs+ ).toBe(45_000)})})@@ -489,7 +513,10 @@ describe('codex-llm-proxy attempt telemetry', () => {return { executionTicket, requestHash, request: raw }}- function grantingClient(): { client: ControlApiClient; receipts: unknown[] } {+ function grantingClient(maxStreamDurationMs = 300_000): {+ client: ControlApiClient+ receipts: unknown[]+ } {const receipts: unknown[] = []const claims = Buffer.from(JSON.stringify({@@ -507,7 +534,7 @@ describe('codex-llm-proxy attempt telemetry', () => {catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',operation: 'completion_stream',servedModel: 'gpt-5.1',- maxStreamDurationMs: 300_000,+ maxStreamDurationMs,},expiryClass: 'short_lived',attemptReceipt: 'a'.repeat(64),@@ -557,6 +584,36 @@ describe('codex-llm-proxy attempt telemetry', () => {} as unknown as ControlApiClient}+ // Sends `textDeltas` frames, then never sends another byte. The body ignores+ // the fetch signal, like a peer that stops writing without closing.+ function stalledUpstream(textDeltas: number): typeof fetch {+ return (async () => {+ const encoder = new TextEncoder()+ let sent = 0+ const body = new ReadableStream<Uint8Array>(+ {+ pull(controller) {+ if (sent < textDeltas) {+ controller.enqueue(+ encoder.encode(+ `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${sent}` })}\n\n`+ )+ )+ sent += 1+ return undefined+ }+ return new Promise<void>(() => undefined)+ },+ },+ { highWaterMark: 0 }+ )+ return new Response(body, {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })+ }) as typeof fetch+ }+function failureCount(metricsText: string, code: string): number {const line = metricsText.split('\n')@@ -564,19 +621,31 @@ describe('codex-llm-proxy attempt telemetry', () => {return line ? Number(line.split(' ').pop()) : 0}+ function timeoutCount(metricsText: string, kind: 'idle' | 'total'): number {+ const line = metricsText+ .split('\n')+ .find(row => row.startsWith(`codex_proxy_upstream_timeouts_total{kind="${kind}"}`))+ return line ? Number(line.split(' ').pop()) : 0+ }+async function run(providerAttemptId: string,textDeltas: number,calls: number,tamper?: (raw: Record<string, unknown>) => void,- deniedCode?: string+ deniedCode?: string,+ options: {+ fetchFn?: typeof fetch+ maxStreamDurationMs?: number+ configOverrides?: Partial<CodexLlmProxyConfig>+ } = {}) {const info = vi.spyOn(logger, 'info')const warn = vi.spyOn(logger, 'warn')- const { client, receipts } = grantingClient()- const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ const { client, receipts } = grantingClient(options.maxStreamDurationMs)+ const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {controlApiClient: deniedCode ? denyingClient(deniedCode) : client,- fetchFn: upstream(textDeltas, calls),+ fetchFn: options.fetchFn ?? upstream(textDeltas, calls),lookup,})try {@@ -641,6 +710,81 @@ describe('codex-llm-proxy attempt telemetry', () => {expectNoForbiddenKeys(lines[0]!)})+ it('(h) answers 504 stream_duration_exceeded when the total cap fires before any frame', async () => {+ const { res, receipts, lines, metricsText } = await run(+ 'att-total-http',+ 0,+ 0,+ undefined,+ undefined,+ { fetchFn: stalledUpstream(0), maxStreamDurationMs: 100 }+ )+ expect(res.status).toBe(504)+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.body).toEqual({ error: 'stream_duration_exceeded' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-total-http',+ outcome: 'failed',+ code: 'stream_duration_exceeded',+ reason: 'upstream stream exceeded maxStreamDurationMs',+ details: { limitMs: 100 },+ deliveredAs: 'http_status',+ httpStatus: 504,+ })+ expectNoForbiddenKeys(lines[0]!)+ expect(failureCount(metricsText, 'stream_duration_exceeded')).toBe(1)+ expect(failureCount(metricsText, 'other')).toBe(0)+ expect(timeoutCount(metricsText, 'total')).toBe(1)+ expect(timeoutCount(metricsText, 'idle')).toBe(0)+ })++ it('(i) sends a stream_duration_exceeded SSE frame when text was already streamed', async () => {+ const { res, lines } = await run('att-total-sse', 0, 0, undefined, undefined, {+ fetchFn: stalledUpstream(1),+ maxStreamDurationMs: 100,+ })+ expect(res.status).toBe(200)+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')+ expect(res.text).toContain('data: {"type":"error","code":"stream_duration_exceeded"}')+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ outcome: 'failed',+ code: 'stream_duration_exceeded',+ deliveredAs: 'sse_error',+ textChunks: 1,+ })+ expect('httpStatus' in lines[0]!).toBe(false)+ })++ it('(j) answers 503 provider_unavailable and counts an idle timeout on a silent upstream', async () => {+ const { res, receipts, lines, metricsText } = await run(+ 'att-idle-http',+ 0,+ 0,+ undefined,+ undefined,+ { fetchFn: stalledUpstream(0), configOverrides: { upstreamIdleTimeoutMs: 50 } }+ )+ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-idle-http',+ outcome: 'failed',+ code: 'provider_unavailable',+ reason: 'upstream stream idle timeout',+ details: { idleTimeoutMs: 50 },+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(timeoutCount(metricsText, 'idle')).toBe(1)+ expect(timeoutCount(metricsText, 'total')).toBe(0)+ })+it('(d) logs an invalid request without the caller-supplied parse message', async () => {const secretField = 'sk-live-0123456789abcdef'const { res, lines } = await run('att-invalid', 0, 0, raw => {diff --git a/codex-llm-proxy/test/sseBackpressure.test.ts b/codex-llm-proxy/test/sseBackpressure.test.tsindex 7902b39b5..3cfb1971e 100644--- a/codex-llm-proxy/test/sseBackpressure.test.ts+++ b/codex-llm-proxy/test/sseBackpressure.test.ts@@ -92,6 +92,7 @@ function config(): CodexLlmProxyConfig {maxBodyBytes: 1_048_576,maxStreamDurationMs: 60_000,maxDeadlineMs: 60_000,+ upstreamIdleTimeoutMs: 300_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.tsindex 0821c473b..21e141065 100644--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts@@ -1,11 +1,38 @@import { describe, expect, it } from 'vitest'-import { readFileSync } from 'node:fs'+import { readdirSync, readFileSync } from 'node:fs'import { STREAM_LIMITS } from '../src/requestLimits.js'const fixturePath = new URL('../../tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json',import.meta.url)+const srcDir = new URL('../src/', import.meta.url)++// Every construction of a transport error in src, with its first code argument.+// UpstreamTimeoutError takes its metric kind first and its wire code second.+const TRANSPORT_ERROR_SITE =+ /new (?:CodexTransportError|UpstreamTimeoutError)\(\s*(?:'(?:idle|total)',\s*)?([^,)\s]+)/g++function emittedTransportCodes(): { codes: Set<string>; sites: number; constructions: number } {+ const codes = new Set<string>()+ let sites = 0+ let constructions = 0+ for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {+ const source = readFileSync(new URL(name, srcDir), 'utf8')+ constructions +=+ source.split('new CodexTransportError(').length - 1 ++ source.split('new UpstreamTimeoutError(').length - 1+ for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {+ const literal = /^'([a-z][a-z0-9_]+)'$/.exec(match[1]!)+ // A computed code cannot be checked against the taxonomy, so it fails here.+ expect(literal, `${name}: transport error code must be a string literal, got ${match[1]}`)+ .not.toBeNull()+ codes.add(literal![1]!)+ sites += 1+ }+ }+ return { codes, sites, constructions }+}describe('codex-subscription stream limits freeze', () => {it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {@@ -26,4 +53,20 @@ describe('codex-subscription stream limits freeze', () => {expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })}})++ it('publishes every transport error code the proxy emits in the fixture errorTaxonomy', () => {+ const { errorTaxonomy } = JSON.parse(readFileSync(fixturePath, 'utf8')) as {+ errorTaxonomy: string[]+ }+ const { codes, sites, constructions } = emittedTransportCodes()+ // Liveness witness: the pattern read the code of every construction a+ // plain substring count finds, including both upstream timeout codes.+ expect(constructions).toBeGreaterThanOrEqual(15)+ expect(sites).toBe(constructions)+ expect([...codes]).toEqual(+ expect.arrayContaining(['provider_unavailable', 'stream_duration_exceeded'])+ )+ const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()+ expect(unpublished, 'emitted transport codes missing from errorTaxonomy').toEqual([])+ })})diff --git a/desktop-app/ui/src/components/agents/__tests__/ChatThread.renderSemantics.test.tsx b/desktop-app/ui/src/components/agents/__tests__/ChatThread.renderSemantics.test.tsxindex b97c4f21b..e2255053e 100644--- a/desktop-app/ui/src/components/agents/__tests__/ChatThread.renderSemantics.test.tsx+++ b/desktop-app/ui/src/components/agents/__tests__/ChatThread.renderSemantics.test.tsx@@ -132,6 +132,12 @@ describe('ChatThread error code labels', () => {expect(text).not.toContain('Model Overloaded')})+ it('labels a stream duration cap as "Response Took Too Long", not "Model Overloaded"', () => {+ const { label, text } = renderErrorLabel('LLM_STREAM_DURATION_EXCEEDED', 'grok-subscription')+ expect(label).toBe('Response Took Too Long · GROK-SUBSCRIPTION')+ expect(text).not.toContain('Model Overloaded')+ })+it('labels a context length error as "Conversation Too Long"', () => {expect(renderErrorLabel('LLM_CONTEXT_LENGTH_EXCEEDED').label).toBe('Conversation Too Long')})diff --git a/desktop-app/ui/src/constants/agents.ts b/desktop-app/ui/src/constants/agents.tsindex b5a58ade5..6281078c0 100644--- a/desktop-app/ui/src/constants/agents.ts+++ b/desktop-app/ui/src/constants/agents.ts@@ -15,6 +15,7 @@ export const AGENT_ERROR_CODE_LABELS: Record<string, string> = {LLM_CONTENT_FILTERED: 'Content Filtered',LLM_API_CALL_FAILED: 'Connection Error',LLM_TOOL_CALL_LIMIT_EXCEEDED: 'Too Many Tool Calls',+ LLM_STREAM_DURATION_EXCEEDED: 'Response Took Too Long',LLM_CONTEXT_LENGTH_EXCEEDED: 'Conversation Too Long',LLM_IMAGE_INPUT_UNSUPPORTED: 'Image Input Unsupported',LLM_IMAGE_INPUT_UNKNOWN: 'Image Input Unverified',diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex 48e56e018..fdb080383 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -241,8 +241,28 @@ behavior changes:Stable codes: `insufficient_scope`, `no_grant`, `model_not_allowed`,`budget_denied`, `connection_unavailable`, `provider_unavailable`,`origin_denied`, `ticket_invalid`, `ticket_replayed`, `request_hash_mismatch`,-`tool_call_limit_exceeded`.-+`invalid_request`, `tool_call_limit_exceeded`, `sse_buffer_exceeded`,+`stream_duration_exceeded`. The freeze gate checks that every code the proxy+constructs is in the fixture's `errorTaxonomy`.++- `invalid_request`: the request body failed the transport schema (HTTP 400+ before redeem), or the upstream answered the completion with HTTP 400.+- `sse_buffer_exceeded`: the upstream sent more than 1 MiB without the blank+ line that ends an SSE event.+- `stream_duration_exceeded`: the attempt reached `maxStreamDurationMs` (the+ total cap, bounded again by the ticket's deadline). The proxy cancels the+ upstream body and returns HTTP 504, or an SSE error frame when text had+ already been streamed. The Host maps it to `LLM_STREAM_DURATION_EXCEEDED`.+ It is not retryable and not failover-eligible: another attempt would spend+ the same budget on the same turn.+- Idle timeout: when the upstream sends no byte for `upstreamIdleTimeoutMs`,+ the proxy cancels the upstream body and fails the attempt with+ `provider_unavailable` (HTTP 503, reason `upstream stream idle timeout`).+ That code stays retryable and failover-eligible, because a silent upstream+ is an outage of that provider, not a property of the turn.+ `CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS` can lower the idle timeout; the+ transport never raises it above the table value. Both cuts are counted in+ `codex_proxy_upstream_timeouts_total{kind="idle"|"total"}`.- `tool_call_limit_exceeded`: the upstream response carried more than`maxToolCalls` tool calls. The proxy returns HTTP 422, or an SSE error framewhen text had already been streamed. The Host maps it todiff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex 175d5ac9e..72d162387 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -234,8 +234,30 @@ Stable codes: `insufficient_scope`, `no_grant`, `model_not_allowed`,`budget_denied` (Host-side only), `connection_unavailable`,`provider_unavailable`, `origin_denied`, `ticket_invalid`, `ticket_replayed`,`request_hash_mismatch`, `client_upgrade_required`, `tool_call_limit_exceeded`,-`tool_call_arguments_exceeded`, `request_limit_exceeded` (Host-side only).-+`tool_call_arguments_exceeded`, `invalid_request`, `sse_buffer_exceeded`,+`stream_duration_exceeded`, `request_limit_exceeded` (Host-side only). The+freeze gate checks that every code the proxy constructs is in the fixture's+`errorTaxonomy`.++- `invalid_request`: the request body failed the transport schema (HTTP 400+ before redeem), or the upstream answered the completion with HTTP 400.+- `sse_buffer_exceeded`: the upstream sent more than 1 MiB without the blank+ line that ends an SSE event.+- `stream_duration_exceeded`: the attempt reached `maxStreamDurationMs` (the+ total cap, bounded again by the ticket's deadline). The proxy cancels the+ upstream body and returns HTTP 504, or an SSE error frame when text had+ already been streamed. The Host maps it to `LLM_STREAM_DURATION_EXCEEDED`.+ It is not retryable and not failover-eligible: another attempt would spend+ the same budget on the same turn.+- Idle timeout: when the upstream sends no byte for `upstreamIdleTimeoutMs`+ (600000, the value read from the Grok Build client source), the proxy cancels the upstream+ body and fails the attempt with `provider_unavailable` (HTTP 503, reason+ `upstream stream idle timeout`). That code stays retryable and+ failover-eligible, because a silent upstream is an outage of that provider,+ not a property of the turn. `GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS` can+ lower the idle timeout; the transport never raises it above the contract+ value. Both cuts are counted in+ `grok_proxy_upstream_timeouts_total{kind="idle"|"total"}`.- `tool_call_limit_exceeded`: the upstream response carried more than`maxToolCalls` tool calls. The proxy returns HTTP 422 whose body is the codealone — `{"error":"tool_call_limit_exceeded"}` — or an SSE error frame when@@ -268,10 +290,10 @@ Stable codes: `insufficient_scope`, `no_grant`, `model_not_allowed`,`grok_proxy_attempt_failures_total{code}` counts failed attempts. Its labelallowlist is `ATTEMPT_ERROR_STATUS` in `grok-llm-proxy/src/server.ts` — thesame table that maps a code to its HTTP status — not the list above. The table-is a superset: it also carries the proxy and control-api codes that never reach-the Host as a provider error (`invalid_request`, `Unauthorized`,-`ticket_expired`, `host_binding_mismatch`, `disabled`, `sse_buffer_exceeded`,-`invalid_receipt`, `conflict`), and it omits the two Host-side codes above.+is a superset: it also carries the control-api codes that never reach the Host+as a provider error (`Unauthorized`, `ticket_expired`, `host_binding_mismatch`,+`disabled`, `invalid_receipt`, `conflict`), and it omits the two Host-side codes+above.Anything outside the table is recorded as `other`, because a control-api errorbody is not bounded by the proxy. The raw code stays in the`grok_proxy_attempt_finished` log line.diff --git a/grok-llm-proxy/src/config.ts b/grok-llm-proxy/src/config.tsindex 92f8d3996..f1c0253dd 100644--- a/grok-llm-proxy/src/config.ts+++ b/grok-llm-proxy/src/config.ts@@ -1,3 +1,5 @@+import { STREAM_LIMITS } from './requestLimits.js'+export type GrokLlmProxyConfig = {runtimePort: numberadminPort: number@@ -5,6 +7,8 @@ export type GrokLlmProxyConfig = {maxBodyBytes: numbermaxStreamDurationMs: numbermaxDeadlineMs: number+ /** Lowers STREAM_LIMITS.upstreamIdleTimeoutMs; the transport never raises it. */+ upstreamIdleTimeoutMs: numberjwtIssuer: stringjwtPublicKey: stringexecutionEnabled: boolean@@ -87,6 +91,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCoenv.GROK_LLM_PROXY_MAX_DEADLINE_MS,300_000),+ upstreamIdleTimeoutMs: requiredPositiveInt(+ 'GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS',+ env.GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,+ STREAM_LIMITS.upstreamIdleTimeoutMs+ ),jwtIssuer: env.GROK_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',jwtPublicKey: requiredPem('GROK_LLM_PROXY_JWT_PUBLIC_KEY', env.GROK_LLM_PROXY_JWT_PUBLIC_KEY),executionEnabled: env.GROK_LLM_PROXY_EXECUTION_ENABLED === 'true',diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.tsindex 7314d2aec..43debdcba 100644--- a/grok-llm-proxy/src/grokTransport.ts+++ b/grok-llm-proxy/src/grokTransport.ts@@ -76,6 +76,23 @@ export class GrokTransportError extends Error {}}+/**+ * The proxy cut the upstream stream on one of its two bounds. The wire code+ * stays that of GrokTransportError (`provider_unavailable` for idle silence,+ * `stream_duration_exceeded` for the total cap); `kind` only labels the metric.+ */+export class UpstreamTimeoutError extends GrokTransportError {+ constructor(+ readonly kind: 'idle' | 'total',+ code: string,+ message: string,+ details: Readonly<Record<string, number | string>>+ ) {+ super(code, message, details)+ this.name = 'UpstreamTimeoutError'+ }+}+export type StreamGrokCompletionInput = {executionTicket: stringrequestHash: string@@ -287,7 +304,8 @@ class UpstreamDeadline {) {this.total = setTimeout(() => {this.controller.abort(- new GrokTransportError(+ new UpstreamTimeoutError(+ 'total','stream_duration_exceeded','upstream stream exceeded maxStreamDurationMs',{ limitMs: this.totalMs, elapsedMs: Date.now() - this.started }@@ -309,7 +327,7 @@ class UpstreamDeadline {signal.throwIfAborted()this.idle = setTimeout(() => {this.controller.abort(- new GrokTransportError('provider_unavailable', 'upstream stream idle timeout', {+ new UpstreamTimeoutError('idle', 'provider_unavailable', 'upstream stream idle timeout', {idleTimeoutMs: this.idleMs,elapsedMs: Date.now() - this.started,})diff --git a/grok-llm-proxy/src/metrics.ts b/grok-llm-proxy/src/metrics.tsindex dbb699933..50590cf30 100644--- a/grok-llm-proxy/src/metrics.ts+++ b/grok-llm-proxy/src/metrics.ts@@ -19,6 +19,12 @@ export function createProxyMetrics(register: Registry) {labelNames: ['code'],registers: [register],})+ const upstreamTimeouts = new Counter({+ name: 'grok_proxy_upstream_timeouts_total',+ help: 'Grok upstream streams cut by the proxy, by bound (idle silence or total duration)',+ labelNames: ['kind'],+ registers: [register],+ })return {observeAttempt(outcome: string, operation: string) {outcomes.inc({ outcome, operation })@@ -26,6 +32,9 @@ export function createProxyMetrics(register: Registry) {observeAttemptFailure(code: string) {attemptFailures.inc({ code })},+ observeUpstreamTimeout(kind: 'idle' | 'total') {+ upstreamTimeouts.inc({ kind })+ },observeStream(durationMs: number) {streamSeconds.observe(durationMs / 1000)},diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.tsindex c3f0d53df..234f1736a 100644--- a/grok-llm-proxy/src/server.ts+++ b/grok-llm-proxy/src/server.ts@@ -13,6 +13,7 @@ import {listGrokModels,streamGrokCompletion,testGrokConnection,+ UpstreamTimeoutError,} from './grokTransport.js'import { logger } from './logger.js'import { createProxyMetrics } from './metrics.js'@@ -187,6 +188,7 @@ export function createProxyApps(request: parsed.data.request,deadlineMs: parsed.data.deadlineMs,maxDeadlineMs: Math.min(config.maxDeadlineMs, config.maxStreamDurationMs),+ upstreamIdleTimeoutMs: config.upstreamIdleTimeoutMs,ticket: {jti: ticket.jti,hostRef: ticket.hostRef,@@ -227,6 +229,7 @@ export function createProxyApps(const mapped = mapError(err)metrics.observeAttempt('error', 'completion_stream')metrics.observeAttemptFailure(failureLabel(mapped.code))+ if (err instanceof UpstreamTimeoutError) metrics.observeUpstreamTimeout(err.kind)const deliveredAs = res.headersSent ? 'sse_error' : 'http_status'logger.warn({@@ -390,6 +393,9 @@ const ATTEMPT_ERROR_STATUS: Record<string, number> = {tool_call_limit_exceeded: 422,tool_call_arguments_exceeded: 422,client_upgrade_required: 426,+ // The attempt ran for its whole stream budget. Retrying the same request+ // would spend the same budget again, so it is a gateway timeout, not 503.+ stream_duration_exceeded: 504,connection_unavailable: 503,provider_unavailable: 503,sse_buffer_exceeded: 503,diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.tsindex 92d92c507..c907016dd 100644--- a/grok-llm-proxy/test/contractFreeze.test.ts+++ b/grok-llm-proxy/test/contractFreeze.test.ts@@ -1,5 +1,5 @@import { describe, expect, it } from 'vitest'-import { readFileSync } from 'node:fs'+import { readdirSync, readFileSync } from 'node:fs'import { dirname, join } from 'node:path'import { fileURLToPath } from 'node:url'import {@@ -64,12 +64,41 @@ type Fixture = {limits: Record<string, unknown>identityHeaders: Record<string, string | boolean>forbiddenOrigins: string[]+ errorTaxonomy: unknown}function readFixture(): Fixture {return JSON.parse(readFileSync(fixturePath, 'utf8')) as Fixture}+const srcDir = join(here, '../src')++// Every construction of a transport error in src, with its first code argument.+// UpstreamTimeoutError takes its metric kind first and its wire code second.+const TRANSPORT_ERROR_SITE =+ /new (?:GrokTransportError|UpstreamTimeoutError)\(\s*(?:'(?:idle|total)',\s*)?([^,)\s]+)/g++function emittedTransportCodes(): { codes: Set<string>; sites: number; constructions: number } {+ const codes = new Set<string>()+ let sites = 0+ let constructions = 0+ for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {+ const source = readFileSync(join(srcDir, name), 'utf8')+ constructions +=+ source.split('new GrokTransportError(').length - 1 ++ source.split('new UpstreamTimeoutError(').length - 1+ for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {+ const literal = /^'([a-z][a-z0-9_]+)'$/.exec(match[1]!)+ // A computed code cannot be checked against the taxonomy, so it fails here.+ expect(literal, `${name}: transport error code must be a string literal, got ${match[1]}`)+ .not.toBeNull()+ codes.add(literal![1]!)+ sites += 1+ }+ }+ return { codes, sites, constructions }+}+describe('grok-subscription contract freeze', () => {it('imports the origin-policy constants the proxy actually enforces', () => {expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)@@ -167,6 +196,40 @@ describe('grok-subscription contract freeze', () => {}})+ it('publishes a well-formed errorTaxonomy with the codes the transport contract names', () => {+ const { errorTaxonomy } = readFixture()+ expect(Array.isArray(errorTaxonomy) && errorTaxonomy.length > 0).toBe(true)+ const codes = errorTaxonomy as unknown[]+ for (const code of codes) {+ expect(typeof code).toBe('string')+ expect(String(code)).toMatch(/^[a-z][a-z0-9_]+$/)+ }+ expect(new Set(codes).size, 'errorTaxonomy must not repeat a code').toBe(codes.length)+ expect(codes).toEqual(+ expect.arrayContaining([+ 'tool_call_limit_exceeded',+ 'tool_call_arguments_exceeded',+ 'client_upgrade_required',+ 'sse_buffer_exceeded',+ 'stream_duration_exceeded',+ ])+ )+ })++ it('publishes every transport error code the proxy emits in the fixture errorTaxonomy', () => {+ const errorTaxonomy = readFixture().errorTaxonomy as string[]+ const { codes, sites, constructions } = emittedTransportCodes()+ // Liveness witness: the pattern read the code of every construction a+ // plain substring count finds, including both upstream timeout codes.+ expect(constructions).toBeGreaterThanOrEqual(15)+ expect(sites).toBe(constructions)+ expect([...codes]).toEqual(+ expect.arrayContaining(['provider_unavailable', 'stream_duration_exceeded'])+ )+ const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()+ expect(unpublished, 'emitted transport codes missing from errorTaxonomy').toEqual([])+ })+it('mirrors the exact OAuth origins control-api dials, including revoke', () => {// control-api owns the OAuth client; the proxy never dials auth.x.ai. Cross-// check the frozen fixture against control-api's exported constants bydiff --git a/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts b/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.tsindex 9de6db4af..10be35337 100644--- a/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts+++ b/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts@@ -78,6 +78,7 @@ function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfigmaxBodyBytes: 1_048_576,maxStreamDurationMs: 30_000,maxDeadlineMs: 30_000,+ upstreamIdleTimeoutMs: 600_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex 41adcd868..a0dc2b079 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -34,6 +34,7 @@ function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfigmaxBodyBytes: 1024,maxStreamDurationMs: 300_000,maxDeadlineMs: 300_000,+ upstreamIdleTimeoutMs: 600_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,@@ -295,6 +296,29 @@ describe('grok-llm-proxy security surface', () => {GROK_LLM_PROXY_MAX_STREAM_DURATION_MS: String(Number.MAX_SAFE_INTEGER),})).toThrow(/bounded positive integer/)+ expect(() =>+ loadConfig({+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '0',+ })+ ).toThrow(/GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS must be a finite integer greater than zero/)+ })++ it('defaults the upstream idle timeout to the published STREAM_LIMITS value', () => {+ const loaded = loadConfig({+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',+ GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',+ })+ expect(loaded.upstreamIdleTimeoutMs).toBe(600_000)+ expect(+ loadConfig({+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,+ GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',+ GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',+ GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '45000',+ }).upstreamIdleTimeoutMs+ ).toBe(45_000)})})@@ -510,7 +534,10 @@ describe('grok-llm-proxy attempt telemetry', () => {return { executionTicket, requestHash, request: raw }}- function grantingClient(): { client: ControlApiClient; receipts: unknown[] } {+ function grantingClient(maxStreamDurationMs = 300_000): {+ client: ControlApiClient+ receipts: unknown[]+ } {const receipts: unknown[] = []const client = {async redeem(): Promise<RedeemAttemptSuccess> {@@ -522,7 +549,7 @@ describe('grok-llm-proxy attempt telemetry', () => {catalogOrigin: GROK_CATALOG_ORIGIN,operation: 'completion_stream',servedModel: 'gpt-5.1',- maxStreamDurationMs: 300_000,+ maxStreamDurationMs,},expiryClass: 'short_lived',attemptReceipt: 'c'.repeat(64),@@ -624,6 +651,36 @@ describe('grok-llm-proxy attempt telemetry', () => {} as unknown as ControlApiClient}+ // Sends `textDeltas` frames, then never sends another byte. The body ignores+ // the fetch signal, like a peer that stops writing without closing.+ function stalledUpstream(textDeltas: number): typeof fetch {+ return (async () => {+ const encoder = new TextEncoder()+ let sent = 0+ const body = new ReadableStream<Uint8Array>(+ {+ pull(controller) {+ if (sent < textDeltas) {+ controller.enqueue(+ encoder.encode(+ `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${sent}` })}\n\n`+ )+ )+ sent += 1+ return undefined+ }+ return new Promise<void>(() => undefined)+ },+ },+ { highWaterMark: 0 }+ )+ return new Response(body, {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })+ }) as typeof fetch+ }+function failureCount(metricsText: string, code: string): number {const line = metricsText.split('\n')@@ -631,6 +688,13 @@ describe('grok-llm-proxy attempt telemetry', () => {return line ? Number(line.split(' ').pop()) : 0}+ function timeoutCount(metricsText: string, kind: 'idle' | 'total'): number {+ const line = metricsText+ .split('\n')+ .find(row => row.startsWith(`grok_proxy_upstream_timeouts_total{kind="${kind}"}`))+ return line ? Number(line.split(' ').pop()) : 0+ }+async function run(options: {providerAttemptId: stringtextDeltas?: number@@ -638,11 +702,13 @@ describe('grok-llm-proxy attempt telemetry', () => {tamper?: (raw: Record<string, unknown>) => voiddeniedCode?: stringfetchFn?: typeof fetch+ maxStreamDurationMs?: number+ configOverrides?: Partial<GrokLlmProxyConfig>}) {const info = vi.spyOn(logger, 'info')const warn = vi.spyOn(logger, 'warn')- const { client, receipts } = grantingClient()- const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ const { client, receipts } = grantingClient(options.maxStreamDurationMs)+ const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {controlApiClient: options.deniedCode ? denyingClient(options.deniedCode) : client,fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),lookup,@@ -825,6 +891,76 @@ describe('grok-llm-proxy attempt telemetry', () => {expectNoForbiddenKeys(lines[0]!)})+ it('(t1) answers 504 stream_duration_exceeded when the total cap fires before any frame', async () => {+ const { res, receipts, lines, metricsText } = await run({+ providerAttemptId: 'att-total-http',+ fetchFn: stalledUpstream(0),+ maxStreamDurationMs: 100,+ })+ expect(res.status).toBe(504)+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.body).toEqual({ error: 'stream_duration_exceeded' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-total-http',+ outcome: 'failed',+ code: 'stream_duration_exceeded',+ reason: 'upstream stream exceeded maxStreamDurationMs',+ details: { limitMs: 100 },+ deliveredAs: 'http_status',+ httpStatus: 504,+ })+ expectNoForbiddenKeys(lines[0]!)+ expect(failureCount(metricsText, 'stream_duration_exceeded')).toBe(1)+ expect(failureCount(metricsText, 'other')).toBe(0)+ expect(timeoutCount(metricsText, 'total')).toBe(1)+ expect(timeoutCount(metricsText, 'idle')).toBe(0)+ })++ it('(t2) sends a stream_duration_exceeded SSE frame when text was already streamed', async () => {+ const { res, lines } = await run({+ providerAttemptId: 'att-total-sse',+ fetchFn: stalledUpstream(1),+ maxStreamDurationMs: 100,+ })+ expect(res.status).toBe(200)+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')+ expect(res.text).toContain('data: {"type":"error","code":"stream_duration_exceeded"}')+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ outcome: 'failed',+ code: 'stream_duration_exceeded',+ deliveredAs: 'sse_error',+ textChunks: 1,+ })+ expect('httpStatus' in lines[0]!).toBe(false)+ })++ it('(t3) answers 503 provider_unavailable and counts an idle timeout on a silent upstream', async () => {+ const { res, receipts, lines, metricsText } = await run({+ providerAttemptId: 'att-idle-http',+ fetchFn: stalledUpstream(0),+ configOverrides: { upstreamIdleTimeoutMs: 50 },+ })+ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'provider_unavailable' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-idle-http',+ outcome: 'failed',+ code: 'provider_unavailable',+ reason: 'upstream stream idle timeout',+ details: { idleTimeoutMs: 50 },+ deliveredAs: 'http_status',+ httpStatus: 503,+ })+ expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)+ expect(timeoutCount(metricsText, 'idle')).toBe(1)+ expect(timeoutCount(metricsText, 'total')).toBe(0)+ })+it('(d) labels the failure metric with a known transport code', async () => {const { res, lines, metricsText } = await run({providerAttemptId: 'att-buffer',diff --git a/grok-llm-proxy/test/sseBackpressure.test.ts b/grok-llm-proxy/test/sseBackpressure.test.tsindex d90235c10..52771b2c8 100644--- a/grok-llm-proxy/test/sseBackpressure.test.ts+++ b/grok-llm-proxy/test/sseBackpressure.test.ts@@ -92,6 +92,7 @@ function config(): GrokLlmProxyConfig {maxBodyBytes: 1_048_576,maxStreamDurationMs: 60_000,maxDeadlineMs: 60_000,+ upstreamIdleTimeoutMs: 600_000,jwtIssuer: 'control-api',jwtPublicKey: publicKey,executionEnabled: true,diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex a366bc26f..cd947ff9e 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.367",+ "version": "0.1.368","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.367",+ "version": "0.1.368","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 957c51bd2..3dc925d7b 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.367",+ "version": "0.1.368","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/agent/__tests__/taskExecutor.test.ts b/mcp-host/src/agent/__tests__/taskExecutor.test.tsindex 708fd68f9..ee99bc8fc 100644--- a/mcp-host/src/agent/__tests__/taskExecutor.test.ts+++ b/mcp-host/src/agent/__tests__/taskExecutor.test.ts@@ -1288,6 +1288,7 @@ describe('TaskExecutor error handling', () => {it.each([[LlmErrorCode.ToolCallLimitExceeded, 'LLM_TOOL_CALL_LIMIT_EXCEEDED', false],+ [LlmErrorCode.StreamDurationExceeded, 'LLM_STREAM_DURATION_EXCEEDED', false],// Witness: the same path keeps an existing provider code unchanged.[LlmErrorCode.ModelOverloaded, 'LLM_MODEL_OVERLOADED', true],] as const)(diff --git a/mcp-host/src/core/errors.ts b/mcp-host/src/core/errors.tsindex a5ca599e2..d77add7ed 100644--- a/mcp-host/src/core/errors.ts+++ b/mcp-host/src/core/errors.ts@@ -33,6 +33,13 @@ export enum LlmErrorCode {ModelOverloaded = 'LLM_MODEL_OVERLOADED',/** One model response asked for more tool calls than the provider contract allows. */ToolCallLimitExceeded = 'LLM_TOOL_CALL_LIMIT_EXCEEDED',+ /**+ * The provider proxy cut one attempt at its total stream duration cap.+ * Terminal: a retry would spend the same budget again, so it is never+ * retryable and never failover-eligible. Idle silence is a different code+ * (`provider_unavailable`), which stays retryable.+ */+ StreamDurationExceeded = 'LLM_STREAM_DURATION_EXCEEDED',/*** Issue #654 — the selected (provider, model) is known NOT to accept the* image carried by this attempt (model evidence `unsupported`, selectiondiff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.tsindex f292d3b8a..8552ec53f 100644--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts@@ -108,6 +108,64 @@ describe('CodexLlmProxyClient', () => {expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()})+ // The proxy's total stream cap arrives as 504 before any frame, or as an SSE+ // error frame after one. The attempt spent its whole budget, so the same+ // request would spend it again elsewhere: no retry, no failover.+ it.each([+ {+ path: '504 JSON body before streaming',+ response: {+ ok: false,+ status: 504,+ json: async () => ({ error: 'stream_duration_exceeded' }),+ },+ message: 'proxy stream failed with 504 (stream_duration_exceeded)',+ },+ {+ path: 'SSE error frame after a text frame',+ response: {+ ok: true,+ body: sse([+ { type: 'text', text: 'partial' },+ { type: 'error', code: 'stream_duration_exceeded' },+ ]),+ },+ message: 'proxy stream failed with stream_duration_exceeded',+ },+ ])('surfaces stream_duration_exceeded from the $path', async ({ response, message }) => {+ const fetchFn = vi.fn().mockResolvedValue(response)+ const client = new CodexLlmProxyClient({+ runtimeUrl: resolveCodexProxyRuntimeUrl(+ 'http://codex-llm-proxy.control-plane.svc.cluster.local:8080'+ ),+ readPlatformJwt: () => 'platform-jwt',+ fetchFn: fetchFn as unknown as typeof fetch,+ })+ const err = await client+ .stream({ executionTicket: 'ticket-123456', requestHash: 'a'.repeat(64), request: {} })+ .then(+ () => undefined,+ (e: unknown) => e+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(CodexProxyError)+ expect(err).toMatchObject({ code: 'stream_duration_exceeded', message })++ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {} as never)+ const classified = provider.classifyError(err)+ expect(classified.code).toBe(LlmErrorCode.StreamDurationExceeded)+ expect(classified.retryable).toBe(false)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ // Witness: the idle cut is an outage on the same classifier and does fail over.+ const idle = provider.classifyError(+ new CodexProxyError(+ 'provider_unavailable',+ 'proxy stream failed with 503 (provider_unavailable)'+ )+ )+ expect(classifyFailoverClass(idle.code, idle.retryable)).toBe('provider_unavailable')+ })+it('refuses a runtime URL that is not absolute', () => {expect(() =>diff --git a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.tsindex fa1269c01..739357322 100644--- a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts+++ b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts@@ -187,6 +187,57 @@ describe('GrokLlmProxyClient', () => {expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()})+ // The proxy's total stream cap arrives as 504 before any frame, or as an SSE+ // error frame after one. The attempt spent its whole budget, so the same+ // request would spend it again elsewhere: no retry, no failover.+ it.each([+ {+ path: '504 JSON body before streaming',+ response: {+ ok: false,+ status: 504,+ json: async () => ({ error: 'stream_duration_exceeded' }),+ },+ message: 'proxy stream failed with 504 (stream_duration_exceeded)',+ },+ {+ path: 'SSE error frame after a text frame',+ response: {+ ok: true,+ body: sse([+ { type: 'text', text: 'partial' },+ { type: 'error', code: 'stream_duration_exceeded' },+ ]),+ },+ message: 'proxy stream failed with stream_duration_exceeded',+ },+ ])('surfaces stream_duration_exceeded from the $path', async ({ response, message }) => {+ const fetchFn = vi.fn().mockResolvedValue(response)+ const err = await client(fetchFn)+ .stream(STREAM_INPUT)+ .then(+ () => undefined,+ (e: unknown) => e+ )+ expect(fetchFn).toHaveBeenCalledTimes(1)+ expect(err).toBeInstanceOf(GrokProxyError)+ expect(err).toMatchObject({ code: 'stream_duration_exceeded', message })++ const provider = new GrokSubscriptionProvider('grok-4.6', {} as never)+ const classified = provider.classifyError(err)+ expect(classified.code).toBe(LlmErrorCode.StreamDurationExceeded)+ expect(classified.retryable).toBe(false)+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()+ // Witness: the idle cut is an outage on the same classifier and does fail over.+ const idle = provider.classifyError(+ new GrokProxyError(+ 'provider_unavailable',+ 'proxy stream failed with 503 (provider_unavailable)'+ )+ )+ expect(classifyFailoverClass(idle.code, idle.retryable)).toBe('provider_unavailable')+ })+it('fails closed when the proxy emits an SSE error frame after headers', async () => {const fetchFn = vi.fn().mockResolvedValue({ok: true,diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.tsindex 77438a20f..6607ebd4b 100644--- a/mcp-host/src/llm/codexSubscription.ts+++ b/mcp-host/src/llm/codexSubscription.ts@@ -190,6 +190,17 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {...(providerDispatched !== undefined ? { providerDispatched } : {}),}}+ if (code === 'stream_duration_exceeded') {+ // Not retryable: the proxy already spent the attempt's whole stream+ // budget. A retry or a failover would spend it again on the same turn.+ return {+ code: LlmErrorCode.StreamDurationExceeded,+ retryable: false,+ message: err instanceof Error ? err.message : String(err),+ providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),+ }+ }if (code === 'request_limit_exceeded') {return {code: LlmErrorCode.ContextLengthExceeded,diff --git a/mcp-host/src/llm/failover/__tests__/classify.test.ts b/mcp-host/src/llm/failover/__tests__/classify.test.tsindex fbcad64fa..7f88f812c 100644--- a/mcp-host/src/llm/failover/__tests__/classify.test.ts+++ b/mcp-host/src/llm/failover/__tests__/classify.test.ts@@ -30,6 +30,13 @@ describe('classifyFailoverClass', () => {expect(classifyFailoverClass(LlmErrorCode.ModelOverloaded, true)).toBe('provider_unavailable')})+ it('never fails over a stream that hit its total duration cap', () => {+ expect(classifyFailoverClass(LlmErrorCode.StreamDurationExceeded, false)).toBeNull()+ expect(classifyFailoverClass(LlmErrorCode.StreamDurationExceeded, true)).toBeNull()+ // Witness: the same classifier still switches on an outage.+ expect(classifyFailoverClass(LlmErrorCode.ModelOverloaded, true)).toBe('provider_unavailable')+ })+it('exports all four classes as the default triggerOn set', () => {expect([...ALL_FAILOVER_CLASSES].sort()).toEqual(['auth', 'insufficient_quota', 'provider_unavailable', 'rate_limited'].sort()diff --git a/mcp-host/src/llm/failover/__tests__/engine.test.ts b/mcp-host/src/llm/failover/__tests__/engine.test.tsindex 2594e091e..193ff062f 100644--- a/mcp-host/src/llm/failover/__tests__/engine.test.ts+++ b/mcp-host/src/llm/failover/__tests__/engine.test.ts@@ -287,6 +287,47 @@ describe('FailoverEngine', () => {})})+ // A stream cut at its total cap spent the attempt's whole budget; the+ // fallback would spend it again on the same request.+ it('propagates a stream duration cap without switching or setting a cooldown', async () => {+ const stream = vi+ .fn()+ .mockRejectedValue(+ new CodexProxyError(+ 'stream_duration_exceeded',+ 'proxy stream failed with 504 (stream_duration_exceeded)'+ )+ )+ const primary = codexPrimary(stream)+ const fallbackBuild = vi.fn(() => () => Promise.resolve('fallback-served'))+ const e = engine()++ const failure = await e+ .run(PRIMARY, t => (t.kind === 'primary' ? primary.call : fallbackBuild()), classify)+ .catch((err: unknown) => err)++ // Liveness witness: the primary ran once and really reached the proxy.+ expect(primary.call).toHaveBeenCalledTimes(1)+ expect(stream).toHaveBeenCalledTimes(1)+ expect(failure).toBe(primary.thrown[0])+ expect(failure).toMatchObject({+ code: LlmErrorCode.StreamDurationExceeded,+ retryable: false,+ })+ expect(fallbackBuild).not.toHaveBeenCalled()+ expect(metricInc).not.toHaveBeenCalled()++ // No cooldown was set: the next call starts at the primary again.+ const nextPrimary = vi.fn(() => Promise.resolve('primary-again'))+ const next = await e.run(+ PRIMARY,+ t => (t.kind === 'primary' ? nextPrimary : () => Promise.resolve('fallback')),+ classify+ )+ expect(next).toBe('primary-again')+ expect(nextPrimary).toHaveBeenCalledTimes(1)+ })+// The same claim for Grok. This is issue #680 in its most direct form:// before the taxonomy, the limit classified as ModelOverloaded/retryable,// failover fired, and the fallback answered from partial tool results.diff --git a/mcp-host/src/llm/failover/classify.ts b/mcp-host/src/llm/failover/classify.tsindex ca0e59571..188c4c4ca 100644--- a/mcp-host/src/llm/failover/classify.ts+++ b/mcp-host/src/llm/failover/classify.ts@@ -32,7 +32,7 @@ export const ALL_FAILOVER_CLASSES: readonly FailoverClass[] = [* - `ApiCallFailed ∧ !retryable` → null (400 / validation / content-policy —* never trigger; masking these would hide bugs)* - everything else (InvalidResponse, ContextLengthExceeded, ContentFiltered,- * ModelNotAvailable, ToolCallLimitExceeded) → null+ * ModelNotAvailable, ToolCallLimitExceeded, StreamDurationExceeded) → null*/export function classifyFailoverClass(code: LlmErrorCode,diff --git a/mcp-host/src/llm/grokSubscription.ts b/mcp-host/src/llm/grokSubscription.tsindex 034a0178c..19309b1ba 100644--- a/mcp-host/src/llm/grokSubscription.ts+++ b/mcp-host/src/llm/grokSubscription.ts@@ -197,6 +197,17 @@ export class GrokSubscriptionProvider implements SingleTurnProvider {...(providerDispatched !== undefined ? { providerDispatched } : {}),}}+ if (code === 'stream_duration_exceeded') {+ // Not retryable: the proxy already spent the attempt's whole stream+ // budget. A retry or a failover would spend it again on the same turn.+ return {+ code: LlmErrorCode.StreamDurationExceeded,+ retryable: false,+ message: err instanceof Error ? err.message : String(err),+ providerCode: code,+ ...(providerDispatched !== undefined ? { providerDispatched } : {}),+ }+ }// Same family as `request_limit_exceeded`, from the response side: the// transport refused a tool call whose arguments overran its size budget.// Treating it as an outage would retry the identical oversized call and,diff --git a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.tsindex e03d859d3..c922ddd4f 100644--- a/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts+++ b/mcp-host/src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts@@ -387,6 +387,48 @@ describe('LlmBridge authorized multi-provider fallback', () => {])})+ it('does not fail over a stream duration cap, while an overload still does', async () => {+ const capped = new FakeProvider(() => Promise.reject(new Error('stream too long')), {+ code: LlmErrorCode.StreamDurationExceeded,+ retryable: false,+ providerCode: 'stream_duration_exceeded',+ providerDispatched: true,+ })+ const unusedFallback = new FakeProvider(() => Promise.resolve(OK))+ const cappedRun = makeBridge({+ [primary.model]: capped,+ [fallback.model]: unusedFallback,+ })++ await expect(cappedRun.bridge.complete(request)).rejects.toMatchObject({+ code: 'provider_unavailable',+ retryable: false,+ })+ expect(capped.completeSingleTurn).toHaveBeenCalledTimes(1)+ expect(cappedRun.providerCalls).toEqual([`${primary.provider}/${primary.model}`])+ expect(unusedFallback.completeSingleTurn).not.toHaveBeenCalled()++ // Witness: the same bridge setup fails over an overloaded primary.+ const overloaded = new FakeProvider(() => Promise.reject(new Error('overloaded')), {+ code: LlmErrorCode.ModelOverloaded,+ retryable: true,+ })+ const servingFallback = new FakeProvider(() => Promise.resolve(OK))+ const overloadRun = makeBridge({+ [primary.model]: overloaded,+ [fallback.model]: servingFallback,+ })++ await expect(overloadRun.bridge.complete(request)).resolves.toMatchObject({+ servedTarget: fallback,+ fallbackUsed: true,+ })+ expect(overloadRun.providerCalls).toEqual([+ `${primary.provider}/${primary.model}`,+ `${fallback.provider}/${fallback.model}`,+ ])+ })+it('does not use auth failures as an implicit fallback trigger', async () => {const first = new FakeProvider(() => Promise.reject(new Error('401')), {code: LlmErrorCode.AuthenticationFailed,diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.jsonindex 456b4755e..f4351b95a 100644--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json@@ -62,7 +62,10 @@"ticket_invalid","ticket_replayed","request_hash_mismatch",- "tool_call_limit_exceeded"+ "invalid_request",+ "tool_call_limit_exceeded",+ "sse_buffer_exceeded",+ "stream_duration_exceeded"],"termsAndTestAccount": {"automationAuthorized": true,diff --git a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.jsonindex 7bfa5aca8..f62446ac3 100644--- a/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json+++ b/tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json@@ -79,7 +79,13 @@"request_hash_mismatch","fingerprint_in_use","reauth_required",- "disabled"+ "disabled",+ "invalid_request",+ "client_upgrade_required",+ "tool_call_limit_exceeded",+ "tool_call_arguments_exceeded",+ "sse_buffer_exceeded",+ "stream_duration_exceeded"],"termsAndTestAccount": {"automationAuthorized": false,diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 85e91fde0..3a9bac504 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.89",+ "version": "1.0.90","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.89",+ "version": "1.0.90","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex 19d45b543..91a0be7f9 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.89",+ "version": "1.0.90","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.tsindex 433a30bbe..c7f6d4963 100644--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts@@ -255,6 +255,11 @@ describe('codex-subscription contract freeze', () => {expect(String(code)).toMatch(/^[a-z][a-z0-9_]+$/)}expect(errors).toContain('tool_call_limit_exceeded')+ expect(errors).toContain('stream_duration_exceeded')+ expect(errors).toContain('sse_buffer_exceeded')+ expect(new Set(errors as unknown[]).size, 'errorTaxonomy must not repeat a code').toBe(+ (errors as unknown[]).length+ )const terms = contract.termsAndTestAccount as Record<string, unknown> | undefinedexpect(terms && typeof terms === 'object').toBe(true)diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex a7a7dfda3..8c3e659d7 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.89",+ "version": "0.1.90","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.89",+ "version": "0.1.90","devDependencies": {"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract","@types/jsonwebtoken": "^9.0.10",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex d004ff9cc..c0e1b212b 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.89",+ "version": "0.1.90","private": true,"type": "module","scripts": {