The open-source agent harness - the runtime layer that turns an LLM into a working agent.
feat: implement timeout for create-turn SSE writes to enhance reliability (#964)
Sujai Kumar Guptad6b1389353f6
diff --git a/.changeset/turn-sse-write-timeout.md b/.changeset/turn-sse-write-timeout.mdnew file mode 100644index 000000000..1f617898a--- /dev/null+++ b/.changeset/turn-sse-write-timeout.md@@ -0,0 +1,5 @@+---+'@truefoundry/trueforge': patch+---++Time out a hung create-turn SSE write so turn drain and subscribe dual-writes keep running.diff --git a/packages/trueforge/.env.example b/packages/trueforge/.env.exampleindex a5f59b940..2692d6e5c 100644--- a/packages/trueforge/.env.example+++ b/packages/trueforge/.env.example@@ -115,6 +115,8 @@ REDIS_URL=redis://localhost:6379# TURN_STREAM_POST_COMPLETION_TTL_SECONDS=300## A subscribe request stays open for 10 minutes by default.# TURN_SUBSCRIBE_TIMEOUT_MS=600000+## Max ms for one create-turn SSE write; after this the client is dropped but the turn keeps running. Default 60000.+# TURN_SSE_STREAM_WRITE_TIMEOUT_MS=60000## Used when STANDALONE=false (ignored in standalone). Public URL of the app## (origin plus optional pathname). Callbacks for server OIDC and MCP OAuth/DCRdiff --git a/packages/trueforge/src/apis/turns.ts b/packages/trueforge/src/apis/turns.tsindex 8b6e1a2c7..788ba8493 100644--- a/packages/trueforge/src/apis/turns.ts+++ b/packages/trueforge/src/apis/turns.ts@@ -23,10 +23,12 @@ import {isFileContentPart,McpConnectionError,newEventId,+ PromiseTimeoutError,rawSandboxId,redisKey,SandboxError,VercelAILLM,+ withTimeout,} from '@truefoundry/trueforge-core/core';import type { Context } from 'hono';import { HTTPException } from 'hono/http-exception';@@ -820,26 +822,40 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {return c.json({ data: toWireTurn(turn.record) }, 200);}- // Stream: same engine; HTTP handler owns writing each event to SSE.+ // Stream: same engine; a hung SSE write is timed out so drain/dual-write continue.const { drainInput } = await beginTurnExecution(turnParams);let shouldWriteToSSEStream = true;return streamSSE(c, async stream => {stream.onAbort(() => {shouldWriteToSSEStream = false;+ deps.logger.info('Create-turn SSE client disconnected, continuing drain without writing', {+ sessionId,+ turnId: drainInput.turnId,+ });});await drainTurnEvents({...drainInput,onEvent: async (event, sequenceNumber) => {if (!stream.closed && !stream.aborted && shouldWriteToSSEStream) {try {- await stream.writeSSE(turnEventSsePayload(event, sequenceNumber));+ await withTimeout(+ stream.writeSSE(turnEventSsePayload(event, sequenceNumber)),+ configuration.TURN_SSE_STREAM_WRITE_TIMEOUT_MS,+ );} catch (error) {- deps.logger.error('SSE stream write error', extractErrorLogFields(error));+ if (error instanceof PromiseTimeoutError) {+ deps.logger.error('SSE stream write timed out', extractErrorLogFields(error));+ } else {+ deps.logger.error('SSE stream write error', extractErrorLogFields(error));+ }shouldWriteToSSEStream = false;}}},});+ if (!shouldWriteToSSEStream && !stream.closed) {+ stream.abort();+ }await stream.close();});} catch (error) {diff --git a/packages/trueforge/src/config.ts b/packages/trueforge/src/config.tsindex c819f8dd8..6b9cfcf6c 100644--- a/packages/trueforge/src/config.ts+++ b/packages/trueforge/src/config.ts@@ -766,6 +766,11 @@ export interface SharedServerConfiguration {* Env: `TURN_SUBSCRIBE_TIMEOUT_MS`. Default 600000.*/TURN_SUBSCRIBE_TIMEOUT_MS: number;+ /**+ * Max ms to wait for one create-turn SSE write before dropping that client+ * and keeping the turn drain running. Env: `TURN_SSE_STREAM_WRITE_TIMEOUT_MS`. Default 60000.+ */+ TURN_SSE_STREAM_WRITE_TIMEOUT_MS: number;/*** Max ms to wait for a peer executor's reply before failing with 424.* Env: `REDIS_REQUEST_REPLY_TIMEOUT_MS`. Default 60000.@@ -1146,6 +1151,11 @@ const shared: SharedServerConfiguration = {raw: getEnv('TURN_SUBSCRIBE_TIMEOUT_MS'),defaultValue: 600_000,}),+ TURN_SSE_STREAM_WRITE_TIMEOUT_MS: parsePositiveInt({+ envKey: 'TURN_SSE_STREAM_WRITE_TIMEOUT_MS',+ raw: getEnv('TURN_SSE_STREAM_WRITE_TIMEOUT_MS'),+ defaultValue: 60_000,+ }),REDIS_REQUEST_REPLY_TIMEOUT_MS: parsePositiveInt({envKey: 'REDIS_REQUEST_REPLY_TIMEOUT_MS',raw: getEnv('REDIS_REQUEST_REPLY_TIMEOUT_MS'),