The open-source agent harness - the runtime layer that turns an LLM into a working agent.
Cancel active turns during graceful shutdown
debajyoti-truefoundry5895b4a89307
diff --git a/docker-compose.yml b/docker-compose.ymlindex e747d64e8..66027ceb8 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -18,6 +18,9 @@ services:# the image. Copy registry-example to registry before starting the service.volumes:- ./packages/server/registry:/app/packages/server/registry:ro+ # Above the app's default GRACEFUL_TIMEOUT_SECONDS (30) so SIGKILL does not+ # cut off turn drain.+ stop_grace_period: 35srestart: unless-stoppedfrontend:diff --git a/packages/harness/src/agent-session/ITurnResourceResolver.ts b/packages/harness/src/agent-session/ITurnResourceResolver.tsindex 20b14d1ba..eb3c7477e 100644--- a/packages/harness/src/agent-session/ITurnResourceResolver.ts+++ b/packages/harness/src/agent-session/ITurnResourceResolver.ts@@ -8,7 +8,7 @@ import type { TurnRecord } from './models/TurnRecord';import type { AgentSpec } from './schemas/agentSpec';/**- * Per-run wiring contract consumed by SessionHandle.run(). Implementations hold+ * Per-run wiring contract consumed by SessionHandle.createTurn(). Implementations hold* per-request state (secrets, providers, caches) freely.*/export interface ITurnResourceResolver<TTurnCustom extends object = Record<string, never>> {@@ -51,7 +51,7 @@ export interface ITurnResourceResolver<TTurnCustom extends object = Record<strin/*** Release per-run resources. Called by the harness at the end of the run:* by TurnHandle.stream() in its finally (AFTER the terminal state write), or by- * SessionHandle.run() when it throws before a TurnHandle exists — so resources acquired+ * SessionHandle.createTurn() when it throws before a TurnHandle exists — so resources acquired* during a failed run (e.g. a sandbox VM) are still released. Best-effort:* the harness logs and swallows errors — close() can never flip a terminal* state or fail the turn. Must be idempotent. The resolver owns cleanupdiff --git a/packages/harness/src/agent-session/SessionHandle.ts b/packages/harness/src/agent-session/SessionHandle.tsindex b92e216d3..57a6de93d 100644--- a/packages/harness/src/agent-session/SessionHandle.ts+++ b/packages/harness/src/agent-session/SessionHandle.ts@@ -1,5 +1,5 @@/**- * Bound session handle: starts turns via {@link SessionHandle.run}.+ * Bound session handle: starts turns via {@link SessionHandle.createTurn}.*/import { ulid } from 'ulid';import type { AgentDefinition } from '../core/runtime/AgentDefinition';@@ -101,7 +101,7 @@ export class SessionHandle<* failure therefore persists nothing. The returned turn is durable with* state { status: 'running' }.*- * On any failure, run() closes the resolver (best-effort) before rethrowing,+ * On any failure, createTurn() closes the resolver (best-effort) before rethrowing,* so resources acquired here (e.g. a sandbox VM) never outlive a failed run.* On success, ownership of resolver.close() passes to TurnHandle.stream().*@@ -109,7 +109,7 @@ export class SessionHandle<* (no parent, even on a non-empty session); string → fork from that turn* (must exist). Concurrent `'auto'` forks both succeed.*/- async run(input: {+ async createTurn(input: {input?: TurnInputItem[] | undefined;/** 'auto'/omitted → session.last_turn_id; null → new root; id → fork from that turn. */previous_turn_id?: string | null | undefined;@@ -137,7 +137,7 @@ export class SessionHandle<const spec = this.session.agent_spec;// Dispose-on-early-failure: any resource acquired below (sandbox handle,- // MCP connections) is owned by run() until the TurnHandle is returned; from then+ // MCP connections) is owned by createTurn() until the TurnHandle is returned; from then// on TurnHandle.stream()'s finally owns resolver.close(). On any throw in this// acquisition phase we close the resolver (best-effort, error swallowed)// and rethrow — same pattern as the gateway's sessionTurnHandler, which@@ -225,7 +225,7 @@ export class SessionHandle<});} catch (error) {await input.resolver.close().catch((closeError: unknown) => {- input.resolver.logger.warn('TurnResourceResolver.close() failed after run() error', {+ input.resolver.logger.warn('TurnResourceResolver.close() failed after createTurn() error', {err: closeError,});});diff --git a/packages/harness/src/agent-session/Sessions.ts b/packages/harness/src/agent-session/Sessions.tsindex a8c13a51a..9d006b122 100644--- a/packages/harness/src/agent-session/Sessions.ts+++ b/packages/harness/src/agent-session/Sessions.ts@@ -1,6 +1,6 @@/*** Storage-only session collection: create / get. Behavior arrives per run via- * the resolver on {@link SessionHandle.run}.+ * the resolver on {@link SessionHandle.createTurn}.*/import { SessionHandle } from './SessionHandle';import type { CreateSessionInput, GetSessionInput, ISessionStore } from './store/ISessionStore';diff --git a/packages/harness/src/agent-session/TurnHandle.ts b/packages/harness/src/agent-session/TurnHandle.tsindex 4b63b4c31..76b552c43 100644--- a/packages/harness/src/agent-session/TurnHandle.ts+++ b/packages/harness/src/agent-session/TurnHandle.ts@@ -30,6 +30,9 @@ function cancellationReasonFromAbortReason(abortReason: unknown): CancellationReif (abortReason === CancellationReason.CancelledForNextTurn) {return CancellationReason.CancelledForNextTurn;}+ if (abortReason === CancellationReason.Abandoned) {+ return CancellationReason.Abandoned;+ }return CancellationReason.ClientCancelled;}@@ -167,7 +170,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {const resolver = this.resolver;const signal = this.signal;if (!orchestrator || !resolver || !signal) {- throw new Error('TurnHandle.stream() is only available on turns returned from SessionHandle.run()');+ throw new Error('TurnHandle.stream() is only available on turns returned from SessionHandle.createTurn()');}let caughtError: Error | undefined;diff --git a/packages/harness/src/agent-session/models/SessionRecord.ts b/packages/harness/src/agent-session/models/SessionRecord.tsindex 65f82273c..e2ebb39a4 100644--- a/packages/harness/src/agent-session/models/SessionRecord.ts+++ b/packages/harness/src/agent-session/models/SessionRecord.ts@@ -10,7 +10,7 @@ import type { AgentSpec } from '../schemas/agentSpec';export interface SessionRecord<TCustom extends object = Record<string, never>> {tenant_name: string;session_id: string;- /** Always hydrated on read. Source of `spec` in SessionHandle.run(). */+ /** Always hydrated on read. Source of `spec` in SessionHandle.createTurn(). */agent_spec: AgentSpec;/*** Wire SessionSchema.title (nullable). Written via updateSession patch ordiff --git a/packages/harness/src/agent-session/schemas/turn.ts b/packages/harness/src/agent-session/schemas/turn.tsindex 0dc71a95c..a3a482be0 100644--- a/packages/harness/src/agent-session/schemas/turn.ts+++ b/packages/harness/src/agent-session/schemas/turn.ts@@ -15,6 +15,8 @@ export enum CancellationReason {ClientCancelled = 'client-cancelled',// Prior turn aborted because the client started a new turn.CancelledForNextTurn = 'cancelled-for-next-turn',+ // Process shutting down (SIGTERM/SIGINT).+ Abandoned = 'abandoned',}export const TurnStateRunningSchema = zdiff --git a/packages/harness/src/agent-session/store/ISessionStore.ts b/packages/harness/src/agent-session/store/ISessionStore.tsindex b6cefd98f..d82f3a685 100644--- a/packages/harness/src/agent-session/store/ISessionStore.ts+++ b/packages/harness/src/agent-session/store/ISessionStore.ts@@ -164,7 +164,7 @@ export interface ISessionStore</*** MUST return SessionRecord with `agent_spec` fully hydrated, even if the backend* only stores a uri/id (or named-agent ref). Hydration is the store's job — harness- * and SessionHandle.run never resolve agents themselves.+ * and SessionHandle.createTurn never resolve agents themselves.* Does **not** bump `last_activity_timestamp_ms` (read path).*/getSession(input: GetSessionInput): Promise<SessionRecord<TSessionCustom> | undefined>;diff --git a/packages/harness/tests/agent-session/capabilityState.test.ts b/packages/harness/tests/agent-session/capabilityState.test.tsindex 9c58fac94..e918e5ad5 100644--- a/packages/harness/tests/agent-session/capabilityState.test.ts+++ b/packages/harness/tests/agent-session/capabilityState.test.ts@@ -67,7 +67,7 @@ describe('capability_state (tfy.plan fixture)', () => {};let loads: unknown[] = [];- const turn1 = await session.run({+ const turn1 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'start' }],previous_turn_id: null,signal: new AbortController().signal,@@ -95,7 +95,7 @@ describe('capability_state (tfy.plan fixture)', () => {expect(loads).toHaveLength(0);loads = [];- const turn2 = await session.run({+ const turn2 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'continue' }],previous_turn_id: 'auto',signal: new AbortController().signal,@@ -136,7 +136,7 @@ describe('capability_state (tfy.plan fixture)', () => {todo: [{ title: 'step', description: 'do it', status: 'done' }],};- const turn1 = await session.run({+ const turn1 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'start' }],previous_turn_id: null,signal: new AbortController().signal,@@ -167,7 +167,7 @@ describe('capability_state (tfy.plan fixture)', () => {close: () => resolver.close(),};- const turn3 = await session.run({+ const turn3 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'no plan' }],previous_turn_id: 'auto',signal: new AbortController().signal,diff --git a/packages/harness/tests/agent-session/sessions.test.ts b/packages/harness/tests/agent-session/sessions.test.tsindex a191e4f17..f5716806a 100644--- a/packages/harness/tests/agent-session/sessions.test.ts+++ b/packages/harness/tests/agent-session/sessions.test.ts@@ -3,7 +3,7 @@ import { Sessions } from '../../src/agent-session/Sessions';import { InMemorySessionStore } from '../../src/agent-session/store/InMemorySessionStore';import { makeAgentSpec, makeTestResolver } from './testHelpers';-describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {+describe('Sessions / SessionHandle / TurnHandle (storage + createTurn)', () => {const tenant = 'tenant-1';it('create/get hydrates agent_spec', async () => {@@ -31,7 +31,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {session_id: 's1',agent_spec: makeAgentSpec(),});- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -60,7 +60,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {session_id: 's1',agent_spec: makeAgentSpec(),});- const t1 = await session.run({+ const t1 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'one' }],previous_turn_id: null,signal: new AbortController().signal,@@ -69,7 +69,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {});expect(t1.custom).toEqual({ n: 1 });- const t2 = await session.run({+ const t2 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'two' }],previous_turn_id: 'auto',signal: new AbortController().signal,@@ -87,7 +87,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {session_id: 's1',agent_spec: makeAgentSpec(),});- const first = await session.run({+ const first = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'one' }],previous_turn_id: null,signal: new AbortController().signal,@@ -97,7 +97,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {void event;// drain}- const root2 = await session.run({+ const root2 = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'fresh root' }],previous_turn_id: null,signal: new AbortController().signal,@@ -117,7 +117,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {agent_spec: makeAgentSpec(),});await expect(- session.run({+ session.createTurn({// Mixed batch — rejected by SessionHandle.toSendBatch / orchestrator validation path.input: [{ type: EventType.USER_MESSAGE, content: 'hi' },@@ -155,7 +155,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {// Failure path: resources acquired before the throw must be released.const closeOnFailure = jest.fn().mockResolvedValue(undefined);await expect(- session.run({+ session.createTurn({// Mixed batch — rejected after sandbox/thread resolution.input: [{ type: EventType.USER_MESSAGE, content: 'hi' },@@ -175,7 +175,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {// Success path: run() must NOT close — TurnHandle.stream()'s finally owns it.const closeOnSuccess = jest.fn().mockResolvedValue(undefined);- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,diff --git a/packages/harness/tests/agent-session/turnStream.test.ts b/packages/harness/tests/agent-session/turnStream.test.tsindex 1bad5ed95..b12823403 100644--- a/packages/harness/tests/agent-session/turnStream.test.ts+++ b/packages/harness/tests/agent-session/turnStream.test.ts@@ -27,7 +27,7 @@ describe('TurnHandle.stream()', () => {it('run commits running turn; stream is sole terminal writer → done', async () => {const { store, session } = await createSession();- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -57,7 +57,7 @@ describe('TurnHandle.stream()', () => {it('background drain reaches terminal done', async () => {const { session } = await createSession();- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -74,7 +74,7 @@ describe('TurnHandle.stream()', () => {it('second stream() call throws', async () => {const { session } = await createSession();- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -97,7 +97,7 @@ describe('TurnHandle.stream()', () => {it('consumer break/abandon without abort writes cancelled ClientCancelled', async () => {const { store, session } = await createSession();let closeCalls = 0;- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -128,7 +128,7 @@ describe('TurnHandle.stream()', () => {it('abort mid-drain writes terminal cancelled', async () => {const { store, session } = await createSession();const controller = new AbortController();- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: controller.signal,@@ -151,10 +151,66 @@ describe('TurnHandle.stream()', () => {expect(stored?.state.status).toBe('cancelled');});+ it('abort with Abandoned persists reason abandoned', async () => {+ const { store, session } = await createSession();+ const controller = new AbortController();+ const turn = await session.createTurn({+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],+ previous_turn_id: null,+ signal: controller.signal,+ resolver: makeTestResolver(),+ });+ controller.abort(CancellationReason.Abandoned);+ for await (const event of turn.stream()) {+ void event;+ }+ expect(turn.state.status).toBe('cancelled');+ if (turn.state.status === 'cancelled') {+ expect(turn.state.reason).toBe(CancellationReason.Abandoned);+ }+ const stored = await store.getTurn({+ tenant_name: tenant,+ session_id: 's1',+ turn_id: turn.id,+ });+ expect(stored?.state.status).toBe('cancelled');+ if (stored?.state.status === 'cancelled') {+ expect(stored.state.reason).toBe(CancellationReason.Abandoned);+ }+ });++ it('abort with ServerExecutionTimeout persists reason server-execution-timeout', async () => {+ const { store, session } = await createSession();+ const controller = new AbortController();+ const turn = await session.createTurn({+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],+ previous_turn_id: null,+ signal: controller.signal,+ resolver: makeTestResolver(),+ });+ controller.abort(CancellationReason.ServerExecutionTimeout);+ for await (const event of turn.stream()) {+ void event;+ }+ expect(turn.state.status).toBe('cancelled');+ if (turn.state.status === 'cancelled') {+ expect(turn.state.reason).toBe(CancellationReason.ServerExecutionTimeout);+ }+ const stored = await store.getTurn({+ tenant_name: tenant,+ session_id: 's1',+ turn_id: turn.id,+ });+ expect(stored?.state.status).toBe('cancelled');+ if (stored?.state.status === 'cancelled') {+ expect(stored.state.reason).toBe(CancellationReason.ServerExecutionTimeout);+ }+ });+it('resolver.close() called once in finally; throwing close does not flip terminal state', async () => {const { store, session } = await createSession();let closeCalls = 0;- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -191,7 +247,7 @@ describe('TurnHandle.stream()', () => {});const { session } = await createSession();// Spec already has sandbox.enabled from createSession helper.- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],previous_turn_id: null,signal: new AbortController().signal,@@ -241,7 +297,7 @@ describe('TurnResourceResolver caches', () => {expect(creates).toBe(1);});- it('resolveSandbox called once per run via SessionHandle.run', async () => {+ it('resolveSandbox called once per run via SessionHandle.createTurn', async () => {const sandbox = makeStubPublicSandbox();jest.spyOn(sandbox, 'close').mockResolvedValue(undefined);let sandboxCreates = 0;@@ -266,7 +322,7 @@ describe('TurnResourceResolver caches', () => {},}),});- const turn = await session.run({+ const turn = await session.createTurn({input: [{ type: EventType.USER_MESSAGE, content: 'hi' }],previous_turn_id: null,signal: new AbortController().signal,diff --git a/packages/server/src/apis/turns.ts b/packages/server/src/apis/turns.tsindex 1eafe7d47..6cab054c2 100644--- a/packages/server/src/apis/turns.ts+++ b/packages/server/src/apis/turns.ts@@ -1,6 +1,7 @@import { OpenAPIHono, type RouteHandler } from '@hono/zod-openapi';import type { Sessions, Turn, TurnRecord, TurnStreamingEvent } from '@truefoundry/utils/agent-session';import {+ CancellationReason,SessionStoreConflictError,SessionStoreNotFoundError,TurnResourceResolver,@@ -16,9 +17,9 @@ import {OpenAILLM,} from '@truefoundry/utils/core';import { streamSSE } from 'hono/streaming';-import { Readable } from 'stream';import type { Logger } from 'winston';-import { createTurnRoute, getTurnRoute, listTurnEventsRoute, listTurnsRoute } from '../routes/turnRoutes';+import configuration from '../config';+import { createAndExecuteTurnRoute, getTurnRoute, listTurnEventsRoute, listTurnsRoute } from '../routes/turnRoutes';import type { ActiveTurnRegistry } from '../runtime/activeTurns';import type { McpStore } from '../store/McpStore';import type { ModelStore } from '../store/ModelStore';@@ -181,7 +182,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {}};- const createTurnHandler: RouteHandler<typeof createTurnRoute> = async c => {+ const createAndExecuteTurnHandler: RouteHandler<typeof createAndExecuteTurnRoute> = async c => {const { sessionId } = c.req.valid('param');const body = c.req.valid('json');@@ -205,7 +206,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {let turn;try {- turn = await session.run({+ turn = await session.createTurn({input: body.input,previous_turn_id: body.previous_turn_id,signal: abortController.signal,@@ -225,11 +226,21 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {throw error;}- deps.activeTurns.register({ sessionId, turnId: turn.id, abortController });+ // Long timer not tied to any await; without unref() it would keep the process alive.+ // Armed only after createTurn succeeds so a 4xx path never leaks a pending timer.+ const maxExecutionTimer = setTimeout(() => {+ if (!abortController.signal.aborted) {+ abortController.abort(CancellationReason.ServerExecutionTimeout);+ }+ }, configuration.SERVER_EXECUTION_TIMEOUT_SECONDS * 1000);+ maxExecutionTimer.unref();- // Buffer the run behind a Readable so a slow or disconnected client cannot- // stall execution; the SSE loop below drains it independently.- const generator = Readable.from(turn.stream(), { objectMode: true, highWaterMark: 128000 });+ const trackedStream = deps.activeTurns.track({+ sessionId,+ turnId: turn.id,+ abortController,+ stream: turn.stream(),+ });let shouldWriteToSSEStream = true;let sequenceNumber = -1;@@ -240,7 +251,10 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {shouldWriteToSSEStream = false;});try {- for await (const event of generator as AsyncIterable<TurnStreamingEvent>) {+ // Slow SSE writes backpressure turn.stream() (and thus agent execution /+ // persistence). Acceptable for now; reintroduce an eager buffer if clients+ // stall turns in practice.+ for await (const event of trackedStream) {sequenceNumber += 1;if (!stream.closed && !stream.aborted && shouldWriteToSSEStream) {try {@@ -254,7 +268,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {} catch (error) {deps.logger.error('Unexpected error in turn SSE stream loop', extractErrorLogFields(error));} finally {- deps.activeTurns.finish({ sessionId, turnId: turn.id });+ clearTimeout(maxExecutionTimer);await stream.close();}});@@ -264,7 +278,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {router.openapi(listTurnsRoute, listTurnsHandler);router.openapi(getTurnRoute, getTurnHandler);router.openapi(listTurnEventsRoute, listTurnEventsHandler);- router.openapi(createTurnRoute, createTurnHandler);+ router.openapi(createAndExecuteTurnRoute, createAndExecuteTurnHandler);// subscribeTurnRoute (routes/turnRoutes.ts) is defined but not registered:// re-subscribing to a running turn needs a live-stream registry that this// single-process server does not have yet.diff --git a/packages/server/src/config.ts b/packages/server/src/config.tsindex 2464332f0..4fe40f01b 100644--- a/packages/server/src/config.ts+++ b/packages/server/src/config.ts@@ -209,6 +209,16 @@ export interface ServerConfiguration {* Env: `SANDBOX_PREVIEW_URL_EXPIRY_SECONDS`. Default 1 hour.*/SANDBOX_PREVIEW_URL_EXPIRY_SECONDS: number;+ /**+ * Max seconds to wait for turn cancellation + connection drain on SIGTERM/SIGINT.+ * Env: `GRACEFUL_TIMEOUT_SECONDS`. Default 30.+ */+ GRACEFUL_TIMEOUT_SECONDS: number;+ /**+ * Max seconds a single turn may execute before it is cancelled with+ * `server-execution-timeout`. Env: `SERVER_EXECUTION_TIMEOUT_SECONDS`. Default 600 (10 minutes).+ */+ SERVER_EXECUTION_TIMEOUT_SECONDS: number;}// ============================================================================@@ -232,6 +242,12 @@ const configuration: ServerConfiguration = {getEnv('SANDBOX_PREVIEW_URL_EXPIRY_SECONDS'),3600,),+ GRACEFUL_TIMEOUT_SECONDS: parsePositiveInt('GRACEFUL_TIMEOUT_SECONDS', getEnv('GRACEFUL_TIMEOUT_SECONDS'), 30),+ SERVER_EXECUTION_TIMEOUT_SECONDS: parsePositiveInt(+ 'SERVER_EXECUTION_TIMEOUT_SECONDS',+ getEnv('SERVER_EXECUTION_TIMEOUT_SECONDS'),+ 600,+ ),} as const;export default configuration;diff --git a/packages/server/src/main.ts b/packages/server/src/main.tsindex 2aefc51b9..45660f3e7 100644--- a/packages/server/src/main.ts+++ b/packages/server/src/main.ts@@ -13,7 +13,7 @@ try {{ ModelStore },{ McpStore },{ SkillStore },- { Sessions, InMemorySessionStore },+ { Sessions, InMemorySessionStore, CancellationReason },{ ActiveTurnRegistry },{ createServerSandboxFactory },] = await Promise.all([@@ -38,13 +38,14 @@ try {// Throws on malformed SANDBOX_SETTINGS; undefined when sandbox is not configured.const skillStore = SkillStore.load();const sandboxFactory = createServerSandboxFactory({ logger });+ const activeTurns = new ActiveTurnRegistry();const app = createServerApp({modelStore: ModelStore.load(),mcpStore: McpStore.load(),skillStore,sessionStore,sessions: new Sessions({ sessionStore }),- activeTurns: new ActiveTurnRegistry(),+ activeTurns,...(sandboxFactory ? { sandboxFactory } : {}),logger,});@@ -58,35 +59,41 @@ try {process.exit(1);});- // Graceful drain: stop accepting new connections, let in-flight requests- // finish, then exit. Without this, Node (running as PID 1 in the container)- // ignores SIGTERM and docker stop escalates to SIGKILL after its grace- // period, killing requests mid-flight.- const DRAIN_TIMEOUT_MS = 8_000;+ // Graceful drain: stop accepting new connections, cancel running turns as+ // abandoned, wait for them to persist terminal state, then exit. Without+ // this, Node (running as PID 1 in the container) ignores SIGTERM and docker+ // stop escalates to SIGKILL after its grace period.let shuttingDown = false;- const shutdown = (signal: NodeJS.Signals) => {+ const shutdown = async (signal: NodeJS.Signals) => {if (shuttingDown) return;shuttingDown = true;logger.info(`Received ${signal}, draining connections before shutdown`);- server.close(() => {- process.exit(0);- });- // Keep-alive connections with no in-flight request would otherwise hold- // the server open until the client closes them.- if ('closeIdleConnections' in server) {- server.closeIdleConnections();- }-- // Force exit if draining outlasts the timeout (kept below Docker's- // default 10s stop grace period so we exit on our own terms).+ // Arm at the start of each shutdown; unref so this timer alone cannot keep the process alive.setTimeout(() => {- logger.warn(`Drain timed out after ${String(DRAIN_TIMEOUT_MS)}ms, exiting`);+ logger.warn(`Drain timed out after ${String(configuration.GRACEFUL_TIMEOUT_SECONDS)}s, exiting`);process.exit(1);- }, DRAIN_TIMEOUT_MS).unref();+ }, configuration.GRACEFUL_TIMEOUT_SECONDS * 1000).unref();++ const closed = new Promise<void>(resolve => {+ server.close(() => {+ resolve();+ });+ });++ // Sets the registry's shutdown reason immediately so late track() (in-flight create-turn still+ // inside session.createTurn) aborts as Abandoned; then drains the registry. await+ // closed covers the gap where the registry is empty before that late track().+ await activeTurns.shutdownAndWait(CancellationReason.Abandoned);+ await closed;+ process.exit(0);};- process.on('SIGTERM', shutdown);- process.on('SIGINT', shutdown);+ process.on('SIGTERM', signal => {+ void shutdown(signal);+ });+ process.on('SIGINT', signal => {+ void shutdown(signal);+ });} catch (error) {console.error('Failed to start server:', error instanceof Error ? error.message : error);process.exit(1);diff --git a/packages/server/src/routes/turnRoutes.ts b/packages/server/src/routes/turnRoutes.tsindex f1fdc1be4..206b025af 100644--- a/packages/server/src/routes/turnRoutes.ts+++ b/packages/server/src/routes/turnRoutes.ts@@ -96,12 +96,12 @@ export const listTurnEventsRoute = createRoute({},});-export const createTurnRoute = createRoute({+export const createAndExecuteTurnRoute = createRoute({method: 'post',path: '/{sessionId}/turns',tags: [SESSIONS_TAG],- summary: 'Create a turn in a session',- description: `Start or continue a turn within a session. Responds with a Server-Sent Events stream.+ summary: 'Create and execute a turn in a session',+ description: `Create a turn within a session and stream its execution as Server-Sent Events.Use \`previous_turn_id\` to chain to the session's last turn (defaults to \`auto\`).`,request: {params: SessionIdParamsSchema,diff --git a/packages/server/src/runtime/activeTurns.test.ts b/packages/server/src/runtime/activeTurns.test.tsnew file mode 100644index 000000000..eccea3675--- /dev/null+++ b/packages/server/src/runtime/activeTurns.test.ts@@ -0,0 +1,211 @@+import { CancellationReason } from '@truefoundry/utils/agent-session';+import assert from 'node:assert/strict';+import { describe, it } from 'node:test';+import { ActiveTurnRegistry } from './activeTurns';++async function* values<T>(items: T[]): AsyncGenerator<T> {+ for (const item of items) {+ yield item;+ }+}++async function* throwingStream(): AsyncGenerator<number> {+ yield 1;+ throw new Error('stream boom');+}++/**+ * Yields once, then waits until `signal` aborts before completing — used to+ * prove shutdownAndWait waits for tracked streams to finish after abort.+ */+async function* gateOnAbort(signal: AbortSignal): AsyncGenerator<string> {+ yield 'started';+ if (signal.aborted) {+ return;+ }+ await new Promise<void>(resolve => {+ signal.addEventListener('abort', () => resolve(), { once: true });+ });+}++describe('ActiveTurnRegistry', () => {+ it('track passes events through and removes the run when the stream completes', async () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ const tracked = registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: values([1, 2, 3]),+ });++ const seen: number[] = [];+ for await (const value of tracked) {+ seen.push(value);+ }+ assert.deepEqual(seen, [1, 2, 3]);+ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.ClientCancelled,+ }),+ false,+ );+ });++ it('track cleans up when the consumer breaks early', async () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ const tracked = registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: values([1, 2, 3]),+ });++ for await (const value of tracked) {+ assert.equal(value, 1);+ break;+ }+ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.ClientCancelled,+ }),+ false,+ );+ });++ it('track cleans up when the stream throws', async () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ const tracked = registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: throwingStream(),+ });++ await assert.rejects(async () => {+ for await (const value of tracked) {+ void value;+ }+ }, /stream boom/);+ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.ClientCancelled,+ }),+ false,+ );+ });++ it('cancelIfRunning aborts with the given reason and returns true', () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ void registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: values([1]),+ });++ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.ClientCancelled,+ }),+ true,+ );+ assert.equal(abortController.signal.aborted, true);+ assert.equal(abortController.signal.reason, CancellationReason.ClientCancelled);+ });++ it('cancelIfRunning returns false for unknown ids', () => {+ const registry = new ActiveTurnRegistry();+ assert.equal(+ registry.cancelIfRunning({+ sessionId: 'missing',+ turnId: 'missing',+ abortReason: CancellationReason.ClientCancelled,+ }),+ false,+ );+ });++ it('cancelIfRunning does not re-abort an already-aborted controller', () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ void registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: values([1]),+ });+ abortController.abort(CancellationReason.ClientCancelled);++ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.Abandoned,+ }),+ true,+ );+ assert.equal(abortController.signal.reason, CancellationReason.ClientCancelled);+ });++ it('shutdownAndWait aborts runs and waits until tracked streams finish', async () => {+ const registry = new ActiveTurnRegistry();+ const abortController = new AbortController();+ const tracked = registry.track({+ sessionId: 's1',+ turnId: 't1',+ abortController,+ stream: gateOnAbort(abortController.signal),+ });++ const drain = (async () => {+ for await (const value of tracked) {+ void value;+ }+ })();++ await registry.shutdownAndWait(CancellationReason.Abandoned);+ assert.equal(abortController.signal.aborted, true);+ assert.equal(abortController.signal.reason, CancellationReason.Abandoned);+ await drain;+ assert.equal(+ registry.cancelIfRunning({+ sessionId: 's1',+ turnId: 't1',+ abortReason: CancellationReason.ClientCancelled,+ }),+ false,+ );+ });++ it('late track after shutdownAndWait aborts immediately with the shutdown reason', async () => {+ const registry = new ActiveTurnRegistry();+ await registry.shutdownAndWait(CancellationReason.Abandoned);++ const abortController = new AbortController();+ const tracked = registry.track({+ sessionId: 's1',+ turnId: 'late',+ abortController,+ stream: values(['x']),+ });++ assert.equal(abortController.signal.aborted, true);+ assert.equal(abortController.signal.reason, CancellationReason.Abandoned);++ for await (const value of tracked) {+ void value;+ }+ });+});diff --git a/packages/server/src/runtime/activeTurns.ts b/packages/server/src/runtime/activeTurns.tsindex 18d288e12..592348b55 100644--- a/packages/server/src/runtime/activeTurns.ts+++ b/packages/server/src/runtime/activeTurns.ts@@ -1,34 +1,71 @@/**- * In-memory registry of turns running in this process, keyed by session. A- * session may have any number of concurrent turns; the registry makes no- * attempt to enforce one turn per session (that would not hold across- * replicas anyway). Cancel code aborts runs through their AbortControllers;+ * In-memory registry of turns executing in this process. Keys are+ * `${sessionId}:${turnId}`. Cancel aborts via AbortController;* TurnHandle.stream() writes the terminal state when the signal fires.+ * `track()` owns registration and cleanup around the stream lifecycle.*/+import { CancellationReason } from '@truefoundry/utils/agent-session';++interface ActiveTurnRun {+ abortController: AbortController;+ waitUntilCompleted: Promise<void>;+ markCompleted: () => void;+}++function activeTurnKey(sessionId: string, turnId: string): string {+ return `${sessionId}:${turnId}`;+}+export class ActiveTurnRegistry {- /** sessionId -> (turnId -> AbortController) */- private readonly runs = new Map<string, Map<string, AbortController>>();-- /** Registers a running turn under its session. */- register(input: { sessionId: string; turnId: string; abortController: AbortController }): void {- let sessionRuns = this.runs.get(input.sessionId);- if (!sessionRuns) {- sessionRuns = new Map();- this.runs.set(input.sessionId, sessionRuns);- }- sessionRuns.set(input.turnId, input.abortController);- }+ private readonly runs = new Map<string, ActiveTurnRun>();+ private alreadyShutDownAbortReason: CancellationReason | undefined;++ /**+ * Registers the run immediately, then returns a generator that forwards+ * `stream` and removes the run when the stream completes (or the consumer+ * exits early). If shutdown has already begun, aborts the controller with+ * the shutdown reason so the turn ends as abandoned.+ */+ track<T>(input: {+ sessionId: string;+ turnId: string;+ abortController: AbortController;+ stream: AsyncIterable<T>;+ }): AsyncGenerator<T> {+ const key = activeTurnKey(input.sessionId, input.turnId);+ const { promise: waitUntilCompleted, resolve } = Promise.withResolvers<undefined>();+ const markCompleted = (): void => {+ resolve(undefined);+ };+ const run: ActiveTurnRun = {+ abortController: input.abortController,+ waitUntilCompleted,+ markCompleted,+ };+ this.runs.set(key, run);- /** Removes the entry once the turn's stream has fully drained. */- finish(input: { sessionId: string; turnId: string }): void {- const sessionRuns = this.runs.get(input.sessionId);- if (!sessionRuns) {- return;+ // Accepted HTTP may still call track() after shutdownAndWait snapshotted;+ // abort immediately so the turn ends as abandoned instead of running past exit.+ if (this.alreadyShutDownAbortReason !== undefined) {+ if (!input.abortController.signal.aborted) {+ input.abortController.abort(this.alreadyShutDownAbortReason);+ }}- sessionRuns.delete(input.turnId);- if (sessionRuns.size === 0) {- this.runs.delete(input.sessionId);++ const complete = () => {+ run.markCompleted();+ if (this.runs.get(key) === run) {+ this.runs.delete(key);+ }+ };+ async function* tracked(): AsyncGenerator<T> {+ try {+ yield* input.stream;+ } finally {+ complete();+ }}+ return tracked();}/**@@ -37,14 +74,33 @@ export class ActiveTurnRegistry {* turn that is not running is a no-op, mirroring the store's* first-terminal-write-wins rule.*/- cancelIfRunning(input: { sessionId: string; turnId: string; abortReason?: string }): boolean {- const abortController = this.runs.get(input.sessionId)?.get(input.turnId);- if (!abortController) {+ cancelIfRunning(input: { sessionId: string; turnId: string; abortReason: CancellationReason }): boolean {+ const run = this.runs.get(activeTurnKey(input.sessionId, input.turnId));+ if (!run) {return false;}- if (!abortController.signal.aborted) {- abortController.abort(input.abortReason);+ if (!run.abortController.signal.aborted) {+ run.abortController.abort(input.abortReason);}return true;}++ /**+ * Enter shutdown mode, abort every run with `abortReason`, and wait until the+ * registry is empty. Late `track()` calls (in-flight HTTP that registered after+ * the first snapshot) are aborted immediately as `abortReason` and included in+ * subsequent wait iterations.+ */+ async shutdownAndWait(abortReason: CancellationReason): Promise<void> {+ this.alreadyShutDownAbortReason = abortReason;+ while (this.runs.size > 0) {+ const pending = Array.from(this.runs.values());+ for (const run of pending) {+ if (!run.abortController.signal.aborted) {+ run.abortController.abort(abortReason);+ }+ }+ await Promise.allSettled(pending.map(run => run.waitUntilCompleted));+ }+ }}