Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

feat: implement timeout for create-turn SSE writes to enhance reliability (#964)

Sujai Kumar Guptad6b1389353f6
diff --git a/.changeset/turn-sse-write-timeout.md b/.changeset/turn-sse-write-timeout.md
new file mode 100644
index 000000000..1f617898a
--- /dev/null
+++ b/.changeset/turn-sse-write-timeout.md
@@ -0,0 +1,5 @@
+---
+'@truefoundry/trueforge': patch
+---
+
+Time out a hung create-turn SSE write so turn drain and subscribe dual-writes keep running.
diff --git a/packages/trueforge/.env.example b/packages/trueforge/.env.example
index a5f59b940..2692d6e5c 100644
--- a/packages/trueforge/.env.example
+++ b/packages/trueforge/.env.example
@@ -115,6 +115,8 @@ REDIS_URL=redis://localhost:6379
# TURN_STREAM_POST_COMPLETION_TTL_SECONDS=300
## A subscribe request stays open for 10 minutes by default.
# TURN_SUBSCRIBE_TIMEOUT_MS=600000
+## Max ms for one create-turn SSE write; after this the client is dropped but the turn keeps running. Default 60000.
+# TURN_SSE_STREAM_WRITE_TIMEOUT_MS=60000
## Used when STANDALONE=false (ignored in standalone). Public URL of the app
## (origin plus optional pathname). Callbacks for server OIDC and MCP OAuth/DCR
diff --git a/packages/trueforge/src/apis/turns.ts b/packages/trueforge/src/apis/turns.ts
index 8b6e1a2c7..788ba8493 100644
--- a/packages/trueforge/src/apis/turns.ts
+++ b/packages/trueforge/src/apis/turns.ts
@@ -23,10 +23,12 @@ import {
isFileContentPart,
McpConnectionError,
newEventId,
+ PromiseTimeoutError,
rawSandboxId,
redisKey,
SandboxError,
VercelAILLM,
+ withTimeout,
} from '@truefoundry/trueforge-core/core';
import type { Context } from 'hono';
import { HTTPException } from 'hono/http-exception';
@@ -820,26 +822,40 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
return c.json({ data: toWireTurn(turn.record) }, 200);
}
- // Stream: same engine; HTTP handler owns writing each event to SSE.
+ // Stream: same engine; a hung SSE write is timed out so drain/dual-write continue.
const { drainInput } = await beginTurnExecution(turnParams);
let shouldWriteToSSEStream = true;
return streamSSE(c, async stream => {
stream.onAbort(() => {
shouldWriteToSSEStream = false;
+ deps.logger.info('Create-turn SSE client disconnected, continuing drain without writing', {
+ sessionId,
+ turnId: drainInput.turnId,
+ });
});
await drainTurnEvents({
...drainInput,
onEvent: async (event, sequenceNumber) => {
if (!stream.closed && !stream.aborted && shouldWriteToSSEStream) {
try {
- await stream.writeSSE(turnEventSsePayload(event, sequenceNumber));
+ await withTimeout(
+ stream.writeSSE(turnEventSsePayload(event, sequenceNumber)),
+ configuration.TURN_SSE_STREAM_WRITE_TIMEOUT_MS,
+ );
} catch (error) {
- deps.logger.error('SSE stream write error', extractErrorLogFields(error));
+ if (error instanceof PromiseTimeoutError) {
+ deps.logger.error('SSE stream write timed out', extractErrorLogFields(error));
+ } else {
+ deps.logger.error('SSE stream write error', extractErrorLogFields(error));
+ }
shouldWriteToSSEStream = false;
}
}
},
});
+ if (!shouldWriteToSSEStream && !stream.closed) {
+ stream.abort();
+ }
await stream.close();
});
} catch (error) {
diff --git a/packages/trueforge/src/config.ts b/packages/trueforge/src/config.ts
index c819f8dd8..6b9cfcf6c 100644
--- a/packages/trueforge/src/config.ts
+++ b/packages/trueforge/src/config.ts
@@ -766,6 +766,11 @@ export interface SharedServerConfiguration {
* Env: `TURN_SUBSCRIBE_TIMEOUT_MS`. Default 600000.
*/
TURN_SUBSCRIBE_TIMEOUT_MS: number;
+ /**
+ * Max ms to wait for one create-turn SSE write before dropping that client
+ * and keeping the turn drain running. Env: `TURN_SSE_STREAM_WRITE_TIMEOUT_MS`. Default 60000.
+ */
+ TURN_SSE_STREAM_WRITE_TIMEOUT_MS: number;
/**
* Max ms to wait for a peer executor's reply before failing with 424.
* Env: `REDIS_REQUEST_REPLY_TIMEOUT_MS`. Default 60000.
@@ -1146,6 +1151,11 @@ const shared: SharedServerConfiguration = {
raw: getEnv('TURN_SUBSCRIBE_TIMEOUT_MS'),
defaultValue: 600_000,
}),
+ TURN_SSE_STREAM_WRITE_TIMEOUT_MS: parsePositiveInt({
+ envKey: 'TURN_SSE_STREAM_WRITE_TIMEOUT_MS',
+ raw: getEnv('TURN_SSE_STREAM_WRITE_TIMEOUT_MS'),
+ defaultValue: 60_000,
+ }),
REDIS_REQUEST_REPLY_TIMEOUT_MS: parsePositiveInt({
envKey: 'REDIS_REQUEST_REPLY_TIMEOUT_MS',
raw: getEnv('REDIS_REQUEST_REPLY_TIMEOUT_MS'),