Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

Cancel active turns during graceful shutdown

debajyoti-truefoundry5895b4a89307
diff --git a/docker-compose.yml b/docker-compose.yml
index e747d64e8..66027ceb8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -18,6 +18,9 @@ services:
# the image. Copy registry-example to registry before starting the service.
volumes:
- ./packages/server/registry:/app/packages/server/registry:ro
+ # Above the app's default GRACEFUL_TIMEOUT_SECONDS (30) so SIGKILL does not
+ # cut off turn drain.
+ stop_grace_period: 35s
restart: unless-stopped
frontend:
diff --git a/packages/harness/src/agent-session/ITurnResourceResolver.ts b/packages/harness/src/agent-session/ITurnResourceResolver.ts
index 20b14d1ba..eb3c7477e 100644
--- a/packages/harness/src/agent-session/ITurnResourceResolver.ts
+++ b/packages/harness/src/agent-session/ITurnResourceResolver.ts
@@ -8,7 +8,7 @@ import type { TurnRecord } from './models/TurnRecord';
import type { AgentSpec } from './schemas/agentSpec';
/**
- * Per-run wiring contract consumed by SessionHandle.run(). Implementations hold
+ * Per-run wiring contract consumed by SessionHandle.createTurn(). Implementations hold
* per-request state (secrets, providers, caches) freely.
*/
export interface ITurnResourceResolver<TTurnCustom extends object = Record<string, never>> {
@@ -51,7 +51,7 @@ export interface ITurnResourceResolver<TTurnCustom extends object = Record<strin
/**
* Release per-run resources. Called by the harness at the end of the run:
* by TurnHandle.stream() in its finally (AFTER the terminal state write), or by
- * SessionHandle.run() when it throws before a TurnHandle exists — so resources acquired
+ * SessionHandle.createTurn() when it throws before a TurnHandle exists — so resources acquired
* during a failed run (e.g. a sandbox VM) are still released. Best-effort:
* the harness logs and swallows errors — close() can never flip a terminal
* state or fail the turn. Must be idempotent. The resolver owns cleanup
diff --git a/packages/harness/src/agent-session/SessionHandle.ts b/packages/harness/src/agent-session/SessionHandle.ts
index b92e216d3..57a6de93d 100644
--- a/packages/harness/src/agent-session/SessionHandle.ts
+++ b/packages/harness/src/agent-session/SessionHandle.ts
@@ -1,5 +1,5 @@
/**
- * Bound session handle: starts turns via {@link SessionHandle.run}.
+ * Bound session handle: starts turns via {@link SessionHandle.createTurn}.
*/
import { ulid } from 'ulid';
import type { AgentDefinition } from '../core/runtime/AgentDefinition';
@@ -101,7 +101,7 @@ export class SessionHandle<
* failure therefore persists nothing. The returned turn is durable with
* state { status: 'running' }.
*
- * On any failure, run() closes the resolver (best-effort) before rethrowing,
+ * On any failure, createTurn() closes the resolver (best-effort) before rethrowing,
* so resources acquired here (e.g. a sandbox VM) never outlive a failed run.
* On success, ownership of resolver.close() passes to TurnHandle.stream().
*
@@ -109,7 +109,7 @@ export class SessionHandle<
* (no parent, even on a non-empty session); string → fork from that turn
* (must exist). Concurrent `'auto'` forks both succeed.
*/
- async run(input: {
+ async createTurn(input: {
input?: TurnInputItem[] | undefined;
/** 'auto'/omitted → session.last_turn_id; null → new root; id → fork from that turn. */
previous_turn_id?: string | null | undefined;
@@ -137,7 +137,7 @@ export class SessionHandle<
const spec = this.session.agent_spec;
// Dispose-on-early-failure: any resource acquired below (sandbox handle,
- // MCP connections) is owned by run() until the TurnHandle is returned; from then
+ // MCP connections) is owned by createTurn() until the TurnHandle is returned; from then
// on TurnHandle.stream()'s finally owns resolver.close(). On any throw in this
// acquisition phase we close the resolver (best-effort, error swallowed)
// and rethrow — same pattern as the gateway's sessionTurnHandler, which
@@ -225,7 +225,7 @@ export class SessionHandle<
});
} catch (error) {
await input.resolver.close().catch((closeError: unknown) => {
- input.resolver.logger.warn('TurnResourceResolver.close() failed after run() error', {
+ input.resolver.logger.warn('TurnResourceResolver.close() failed after createTurn() error', {
err: closeError,
});
});
diff --git a/packages/harness/src/agent-session/Sessions.ts b/packages/harness/src/agent-session/Sessions.ts
index a8c13a51a..9d006b122 100644
--- a/packages/harness/src/agent-session/Sessions.ts
+++ b/packages/harness/src/agent-session/Sessions.ts
@@ -1,6 +1,6 @@
/**
* Storage-only session collection: create / get. Behavior arrives per run via
- * the resolver on {@link SessionHandle.run}.
+ * the resolver on {@link SessionHandle.createTurn}.
*/
import { SessionHandle } from './SessionHandle';
import type { CreateSessionInput, GetSessionInput, ISessionStore } from './store/ISessionStore';
diff --git a/packages/harness/src/agent-session/TurnHandle.ts b/packages/harness/src/agent-session/TurnHandle.ts
index 4b63b4c31..76b552c43 100644
--- a/packages/harness/src/agent-session/TurnHandle.ts
+++ b/packages/harness/src/agent-session/TurnHandle.ts
@@ -30,6 +30,9 @@ function cancellationReasonFromAbortReason(abortReason: unknown): CancellationRe
if (abortReason === CancellationReason.CancelledForNextTurn) {
return CancellationReason.CancelledForNextTurn;
}
+ if (abortReason === CancellationReason.Abandoned) {
+ return CancellationReason.Abandoned;
+ }
return CancellationReason.ClientCancelled;
}
@@ -167,7 +170,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
const resolver = this.resolver;
const signal = this.signal;
if (!orchestrator || !resolver || !signal) {
- throw new Error('TurnHandle.stream() is only available on turns returned from SessionHandle.run()');
+ throw new Error('TurnHandle.stream() is only available on turns returned from SessionHandle.createTurn()');
}
let caughtError: Error | undefined;
diff --git a/packages/harness/src/agent-session/models/SessionRecord.ts b/packages/harness/src/agent-session/models/SessionRecord.ts
index 65f82273c..e2ebb39a4 100644
--- a/packages/harness/src/agent-session/models/SessionRecord.ts
+++ b/packages/harness/src/agent-session/models/SessionRecord.ts
@@ -10,7 +10,7 @@ import type { AgentSpec } from '../schemas/agentSpec';
export interface SessionRecord<TCustom extends object = Record<string, never>> {
tenant_name: string;
session_id: string;
- /** Always hydrated on read. Source of `spec` in SessionHandle.run(). */
+ /** Always hydrated on read. Source of `spec` in SessionHandle.createTurn(). */
agent_spec: AgentSpec;
/**
* Wire SessionSchema.title (nullable). Written via updateSession patch or
diff --git a/packages/harness/src/agent-session/schemas/turn.ts b/packages/harness/src/agent-session/schemas/turn.ts
index 0dc71a95c..a3a482be0 100644
--- a/packages/harness/src/agent-session/schemas/turn.ts
+++ b/packages/harness/src/agent-session/schemas/turn.ts
@@ -15,6 +15,8 @@ export enum CancellationReason {
ClientCancelled = 'client-cancelled',
// Prior turn aborted because the client started a new turn.
CancelledForNextTurn = 'cancelled-for-next-turn',
+ // Process shutting down (SIGTERM/SIGINT).
+ Abandoned = 'abandoned',
}
export const TurnStateRunningSchema = z
diff --git a/packages/harness/src/agent-session/store/ISessionStore.ts b/packages/harness/src/agent-session/store/ISessionStore.ts
index b6cefd98f..d82f3a685 100644
--- a/packages/harness/src/agent-session/store/ISessionStore.ts
+++ b/packages/harness/src/agent-session/store/ISessionStore.ts
@@ -164,7 +164,7 @@ export interface ISessionStore<
/**
* MUST return SessionRecord with `agent_spec` fully hydrated, even if the backend
* only stores a uri/id (or named-agent ref). Hydration is the store's job — harness
- * and SessionHandle.run never resolve agents themselves.
+ * and SessionHandle.createTurn never resolve agents themselves.
* Does **not** bump `last_activity_timestamp_ms` (read path).
*/
getSession(input: GetSessionInput): Promise<SessionRecord<TSessionCustom> | undefined>;
diff --git a/packages/harness/tests/agent-session/capabilityState.test.ts b/packages/harness/tests/agent-session/capabilityState.test.ts
index 9c58fac94..e918e5ad5 100644
--- a/packages/harness/tests/agent-session/capabilityState.test.ts
+++ b/packages/harness/tests/agent-session/capabilityState.test.ts
@@ -67,7 +67,7 @@ describe('capability_state (tfy.plan fixture)', () => {
};
let loads: unknown[] = [];
- const turn1 = await session.run({
+ const turn1 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'start' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -95,7 +95,7 @@ describe('capability_state (tfy.plan fixture)', () => {
expect(loads).toHaveLength(0);
loads = [];
- const turn2 = await session.run({
+ const turn2 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'continue' }],
previous_turn_id: 'auto',
signal: new AbortController().signal,
@@ -136,7 +136,7 @@ describe('capability_state (tfy.plan fixture)', () => {
todo: [{ title: 'step', description: 'do it', status: 'done' }],
};
- const turn1 = await session.run({
+ const turn1 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'start' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -167,7 +167,7 @@ describe('capability_state (tfy.plan fixture)', () => {
close: () => resolver.close(),
};
- const turn3 = await session.run({
+ const turn3 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'no plan' }],
previous_turn_id: 'auto',
signal: new AbortController().signal,
diff --git a/packages/harness/tests/agent-session/sessions.test.ts b/packages/harness/tests/agent-session/sessions.test.ts
index a191e4f17..f5716806a 100644
--- a/packages/harness/tests/agent-session/sessions.test.ts
+++ b/packages/harness/tests/agent-session/sessions.test.ts
@@ -3,7 +3,7 @@ import { Sessions } from '../../src/agent-session/Sessions';
import { InMemorySessionStore } from '../../src/agent-session/store/InMemorySessionStore';
import { makeAgentSpec, makeTestResolver } from './testHelpers';
-describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
+describe('Sessions / SessionHandle / TurnHandle (storage + createTurn)', () => {
const tenant = 'tenant-1';
it('create/get hydrates agent_spec', async () => {
@@ -31,7 +31,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
session_id: 's1',
agent_spec: makeAgentSpec(),
});
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -60,7 +60,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
session_id: 's1',
agent_spec: makeAgentSpec(),
});
- const t1 = await session.run({
+ const t1 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'one' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -69,7 +69,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
});
expect(t1.custom).toEqual({ n: 1 });
- const t2 = await session.run({
+ const t2 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'two' }],
previous_turn_id: 'auto',
signal: new AbortController().signal,
@@ -87,7 +87,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
session_id: 's1',
agent_spec: makeAgentSpec(),
});
- const first = await session.run({
+ const first = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'one' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -97,7 +97,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
void event;
// drain
}
- const root2 = await session.run({
+ const root2 = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'fresh root' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -117,7 +117,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
agent_spec: makeAgentSpec(),
});
await expect(
- session.run({
+ session.createTurn({
// Mixed batch — rejected by SessionHandle.toSendBatch / orchestrator validation path.
input: [
{ type: EventType.USER_MESSAGE, content: 'hi' },
@@ -155,7 +155,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
// Failure path: resources acquired before the throw must be released.
const closeOnFailure = jest.fn().mockResolvedValue(undefined);
await expect(
- session.run({
+ session.createTurn({
// Mixed batch — rejected after sandbox/thread resolution.
input: [
{ type: EventType.USER_MESSAGE, content: 'hi' },
@@ -175,7 +175,7 @@ describe('Sessions / SessionHandle / TurnHandle (storage + run)', () => {
// Success path: run() must NOT close — TurnHandle.stream()'s finally owns it.
const closeOnSuccess = jest.fn().mockResolvedValue(undefined);
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
diff --git a/packages/harness/tests/agent-session/turnStream.test.ts b/packages/harness/tests/agent-session/turnStream.test.ts
index 1bad5ed95..b12823403 100644
--- a/packages/harness/tests/agent-session/turnStream.test.ts
+++ b/packages/harness/tests/agent-session/turnStream.test.ts
@@ -27,7 +27,7 @@ describe('TurnHandle.stream()', () => {
it('run commits running turn; stream is sole terminal writer → done', async () => {
const { store, session } = await createSession();
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -57,7 +57,7 @@ describe('TurnHandle.stream()', () => {
it('background drain reaches terminal done', async () => {
const { session } = await createSession();
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -74,7 +74,7 @@ describe('TurnHandle.stream()', () => {
it('second stream() call throws', async () => {
const { session } = await createSession();
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -97,7 +97,7 @@ describe('TurnHandle.stream()', () => {
it('consumer break/abandon without abort writes cancelled ClientCancelled', async () => {
const { store, session } = await createSession();
let closeCalls = 0;
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -128,7 +128,7 @@ describe('TurnHandle.stream()', () => {
it('abort mid-drain writes terminal cancelled', async () => {
const { store, session } = await createSession();
const controller = new AbortController();
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: controller.signal,
@@ -151,10 +151,66 @@ describe('TurnHandle.stream()', () => {
expect(stored?.state.status).toBe('cancelled');
});
+ it('abort with Abandoned persists reason abandoned', async () => {
+ const { store, session } = await createSession();
+ const controller = new AbortController();
+ const turn = await session.createTurn({
+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ previous_turn_id: null,
+ signal: controller.signal,
+ resolver: makeTestResolver(),
+ });
+ controller.abort(CancellationReason.Abandoned);
+ for await (const event of turn.stream()) {
+ void event;
+ }
+ expect(turn.state.status).toBe('cancelled');
+ if (turn.state.status === 'cancelled') {
+ expect(turn.state.reason).toBe(CancellationReason.Abandoned);
+ }
+ const stored = await store.getTurn({
+ tenant_name: tenant,
+ session_id: 's1',
+ turn_id: turn.id,
+ });
+ expect(stored?.state.status).toBe('cancelled');
+ if (stored?.state.status === 'cancelled') {
+ expect(stored.state.reason).toBe(CancellationReason.Abandoned);
+ }
+ });
+
+ it('abort with ServerExecutionTimeout persists reason server-execution-timeout', async () => {
+ const { store, session } = await createSession();
+ const controller = new AbortController();
+ const turn = await session.createTurn({
+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ previous_turn_id: null,
+ signal: controller.signal,
+ resolver: makeTestResolver(),
+ });
+ controller.abort(CancellationReason.ServerExecutionTimeout);
+ for await (const event of turn.stream()) {
+ void event;
+ }
+ expect(turn.state.status).toBe('cancelled');
+ if (turn.state.status === 'cancelled') {
+ expect(turn.state.reason).toBe(CancellationReason.ServerExecutionTimeout);
+ }
+ const stored = await store.getTurn({
+ tenant_name: tenant,
+ session_id: 's1',
+ turn_id: turn.id,
+ });
+ expect(stored?.state.status).toBe('cancelled');
+ if (stored?.state.status === 'cancelled') {
+ expect(stored.state.reason).toBe(CancellationReason.ServerExecutionTimeout);
+ }
+ });
+
it('resolver.close() called once in finally; throwing close does not flip terminal state', async () => {
const { store, session } = await createSession();
let closeCalls = 0;
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -191,7 +247,7 @@ describe('TurnHandle.stream()', () => {
});
const { session } = await createSession();
// Spec already has sandbox.enabled from createSession helper.
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
previous_turn_id: null,
signal: new AbortController().signal,
@@ -241,7 +297,7 @@ describe('TurnResourceResolver caches', () => {
expect(creates).toBe(1);
});
- it('resolveSandbox called once per run via SessionHandle.run', async () => {
+ it('resolveSandbox called once per run via SessionHandle.createTurn', async () => {
const sandbox = makeStubPublicSandbox();
jest.spyOn(sandbox, 'close').mockResolvedValue(undefined);
let sandboxCreates = 0;
@@ -266,7 +322,7 @@ describe('TurnResourceResolver caches', () => {
},
}),
});
- const turn = await session.run({
+ const turn = await session.createTurn({
input: [{ type: EventType.USER_MESSAGE, content: 'hi' }],
previous_turn_id: null,
signal: new AbortController().signal,
diff --git a/packages/server/src/apis/turns.ts b/packages/server/src/apis/turns.ts
index 1eafe7d47..6cab054c2 100644
--- a/packages/server/src/apis/turns.ts
+++ b/packages/server/src/apis/turns.ts
@@ -1,6 +1,7 @@
import { OpenAPIHono, type RouteHandler } from '@hono/zod-openapi';
import type { Sessions, Turn, TurnRecord, TurnStreamingEvent } from '@truefoundry/utils/agent-session';
import {
+ CancellationReason,
SessionStoreConflictError,
SessionStoreNotFoundError,
TurnResourceResolver,
@@ -16,9 +17,9 @@ import {
OpenAILLM,
} from '@truefoundry/utils/core';
import { streamSSE } from 'hono/streaming';
-import { Readable } from 'stream';
import type { Logger } from 'winston';
-import { createTurnRoute, getTurnRoute, listTurnEventsRoute, listTurnsRoute } from '../routes/turnRoutes';
+import configuration from '../config';
+import { createAndExecuteTurnRoute, getTurnRoute, listTurnEventsRoute, listTurnsRoute } from '../routes/turnRoutes';
import type { ActiveTurnRegistry } from '../runtime/activeTurns';
import type { McpStore } from '../store/McpStore';
import type { ModelStore } from '../store/ModelStore';
@@ -181,7 +182,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
}
};
- const createTurnHandler: RouteHandler<typeof createTurnRoute> = async c => {
+ const createAndExecuteTurnHandler: RouteHandler<typeof createAndExecuteTurnRoute> = async c => {
const { sessionId } = c.req.valid('param');
const body = c.req.valid('json');
@@ -205,7 +206,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
let turn;
try {
- turn = await session.run({
+ turn = await session.createTurn({
input: body.input,
previous_turn_id: body.previous_turn_id,
signal: abortController.signal,
@@ -225,11 +226,21 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
throw error;
}
- deps.activeTurns.register({ sessionId, turnId: turn.id, abortController });
+ // Long timer not tied to any await; without unref() it would keep the process alive.
+ // Armed only after createTurn succeeds so a 4xx path never leaks a pending timer.
+ const maxExecutionTimer = setTimeout(() => {
+ if (!abortController.signal.aborted) {
+ abortController.abort(CancellationReason.ServerExecutionTimeout);
+ }
+ }, configuration.SERVER_EXECUTION_TIMEOUT_SECONDS * 1000);
+ maxExecutionTimer.unref();
- // Buffer the run behind a Readable so a slow or disconnected client cannot
- // stall execution; the SSE loop below drains it independently.
- const generator = Readable.from(turn.stream(), { objectMode: true, highWaterMark: 128000 });
+ const trackedStream = deps.activeTurns.track({
+ sessionId,
+ turnId: turn.id,
+ abortController,
+ stream: turn.stream(),
+ });
let shouldWriteToSSEStream = true;
let sequenceNumber = -1;
@@ -240,7 +251,10 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
shouldWriteToSSEStream = false;
});
try {
- for await (const event of generator as AsyncIterable<TurnStreamingEvent>) {
+ // Slow SSE writes backpressure turn.stream() (and thus agent execution /
+ // persistence). Acceptable for now; reintroduce an eager buffer if clients
+ // stall turns in practice.
+ for await (const event of trackedStream) {
sequenceNumber += 1;
if (!stream.closed && !stream.aborted && shouldWriteToSSEStream) {
try {
@@ -254,7 +268,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
} catch (error) {
deps.logger.error('Unexpected error in turn SSE stream loop', extractErrorLogFields(error));
} finally {
- deps.activeTurns.finish({ sessionId, turnId: turn.id });
+ clearTimeout(maxExecutionTimer);
await stream.close();
}
});
@@ -264,7 +278,7 @@ export function createTurnsRouter(deps: TurnsRouterDeps) {
router.openapi(listTurnsRoute, listTurnsHandler);
router.openapi(getTurnRoute, getTurnHandler);
router.openapi(listTurnEventsRoute, listTurnEventsHandler);
- router.openapi(createTurnRoute, createTurnHandler);
+ router.openapi(createAndExecuteTurnRoute, createAndExecuteTurnHandler);
// subscribeTurnRoute (routes/turnRoutes.ts) is defined but not registered:
// re-subscribing to a running turn needs a live-stream registry that this
// single-process server does not have yet.
diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts
index 2464332f0..4fe40f01b 100644
--- a/packages/server/src/config.ts
+++ b/packages/server/src/config.ts
@@ -209,6 +209,16 @@ export interface ServerConfiguration {
* Env: `SANDBOX_PREVIEW_URL_EXPIRY_SECONDS`. Default 1 hour.
*/
SANDBOX_PREVIEW_URL_EXPIRY_SECONDS: number;
+ /**
+ * Max seconds to wait for turn cancellation + connection drain on SIGTERM/SIGINT.
+ * Env: `GRACEFUL_TIMEOUT_SECONDS`. Default 30.
+ */
+ GRACEFUL_TIMEOUT_SECONDS: number;
+ /**
+ * Max seconds a single turn may execute before it is cancelled with
+ * `server-execution-timeout`. Env: `SERVER_EXECUTION_TIMEOUT_SECONDS`. Default 600 (10 minutes).
+ */
+ SERVER_EXECUTION_TIMEOUT_SECONDS: number;
}
// ============================================================================
@@ -232,6 +242,12 @@ const configuration: ServerConfiguration = {
getEnv('SANDBOX_PREVIEW_URL_EXPIRY_SECONDS'),
3600,
),
+ GRACEFUL_TIMEOUT_SECONDS: parsePositiveInt('GRACEFUL_TIMEOUT_SECONDS', getEnv('GRACEFUL_TIMEOUT_SECONDS'), 30),
+ SERVER_EXECUTION_TIMEOUT_SECONDS: parsePositiveInt(
+ 'SERVER_EXECUTION_TIMEOUT_SECONDS',
+ getEnv('SERVER_EXECUTION_TIMEOUT_SECONDS'),
+ 600,
+ ),
} as const;
export default configuration;
diff --git a/packages/server/src/main.ts b/packages/server/src/main.ts
index 2aefc51b9..45660f3e7 100644
--- a/packages/server/src/main.ts
+++ b/packages/server/src/main.ts
@@ -13,7 +13,7 @@ try {
{ ModelStore },
{ McpStore },
{ SkillStore },
- { Sessions, InMemorySessionStore },
+ { Sessions, InMemorySessionStore, CancellationReason },
{ ActiveTurnRegistry },
{ createServerSandboxFactory },
] = await Promise.all([
@@ -38,13 +38,14 @@ try {
// Throws on malformed SANDBOX_SETTINGS; undefined when sandbox is not configured.
const skillStore = SkillStore.load();
const sandboxFactory = createServerSandboxFactory({ logger });
+ const activeTurns = new ActiveTurnRegistry();
const app = createServerApp({
modelStore: ModelStore.load(),
mcpStore: McpStore.load(),
skillStore,
sessionStore,
sessions: new Sessions({ sessionStore }),
- activeTurns: new ActiveTurnRegistry(),
+ activeTurns,
...(sandboxFactory ? { sandboxFactory } : {}),
logger,
});
@@ -58,35 +59,41 @@ try {
process.exit(1);
});
- // Graceful drain: stop accepting new connections, let in-flight requests
- // finish, then exit. Without this, Node (running as PID 1 in the container)
- // ignores SIGTERM and docker stop escalates to SIGKILL after its grace
- // period, killing requests mid-flight.
- const DRAIN_TIMEOUT_MS = 8_000;
+ // Graceful drain: stop accepting new connections, cancel running turns as
+ // abandoned, wait for them to persist terminal state, then exit. Without
+ // this, Node (running as PID 1 in the container) ignores SIGTERM and docker
+ // stop escalates to SIGKILL after its grace period.
let shuttingDown = false;
- const shutdown = (signal: NodeJS.Signals) => {
+ const shutdown = async (signal: NodeJS.Signals) => {
if (shuttingDown) return;
shuttingDown = true;
logger.info(`Received ${signal}, draining connections before shutdown`);
- server.close(() => {
- process.exit(0);
- });
- // Keep-alive connections with no in-flight request would otherwise hold
- // the server open until the client closes them.
- if ('closeIdleConnections' in server) {
- server.closeIdleConnections();
- }
-
- // Force exit if draining outlasts the timeout (kept below Docker's
- // default 10s stop grace period so we exit on our own terms).
+ // Arm at the start of each shutdown; unref so this timer alone cannot keep the process alive.
setTimeout(() => {
- logger.warn(`Drain timed out after ${String(DRAIN_TIMEOUT_MS)}ms, exiting`);
+ logger.warn(`Drain timed out after ${String(configuration.GRACEFUL_TIMEOUT_SECONDS)}s, exiting`);
process.exit(1);
- }, DRAIN_TIMEOUT_MS).unref();
+ }, configuration.GRACEFUL_TIMEOUT_SECONDS * 1000).unref();
+
+ const closed = new Promise<void>(resolve => {
+ server.close(() => {
+ resolve();
+ });
+ });
+
+ // Sets the registry's shutdown reason immediately so late track() (in-flight create-turn still
+ // inside session.createTurn) aborts as Abandoned; then drains the registry. await
+ // closed covers the gap where the registry is empty before that late track().
+ await activeTurns.shutdownAndWait(CancellationReason.Abandoned);
+ await closed;
+ process.exit(0);
};
- process.on('SIGTERM', shutdown);
- process.on('SIGINT', shutdown);
+ process.on('SIGTERM', signal => {
+ void shutdown(signal);
+ });
+ process.on('SIGINT', signal => {
+ void shutdown(signal);
+ });
} catch (error) {
console.error('Failed to start server:', error instanceof Error ? error.message : error);
process.exit(1);
diff --git a/packages/server/src/routes/turnRoutes.ts b/packages/server/src/routes/turnRoutes.ts
index f1fdc1be4..206b025af 100644
--- a/packages/server/src/routes/turnRoutes.ts
+++ b/packages/server/src/routes/turnRoutes.ts
@@ -96,12 +96,12 @@ export const listTurnEventsRoute = createRoute({
},
});
-export const createTurnRoute = createRoute({
+export const createAndExecuteTurnRoute = createRoute({
method: 'post',
path: '/{sessionId}/turns',
tags: [SESSIONS_TAG],
- summary: 'Create a turn in a session',
- description: `Start or continue a turn within a session. Responds with a Server-Sent Events stream.
+ summary: 'Create and execute a turn in a session',
+ description: `Create a turn within a session and stream its execution as Server-Sent Events.
Use \`previous_turn_id\` to chain to the session's last turn (defaults to \`auto\`).`,
request: {
params: SessionIdParamsSchema,
diff --git a/packages/server/src/runtime/activeTurns.test.ts b/packages/server/src/runtime/activeTurns.test.ts
new file mode 100644
index 000000000..eccea3675
--- /dev/null
+++ b/packages/server/src/runtime/activeTurns.test.ts
@@ -0,0 +1,211 @@
+import { CancellationReason } from '@truefoundry/utils/agent-session';
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import { ActiveTurnRegistry } from './activeTurns';
+
+async function* values<T>(items: T[]): AsyncGenerator<T> {
+ for (const item of items) {
+ yield item;
+ }
+}
+
+async function* throwingStream(): AsyncGenerator<number> {
+ yield 1;
+ throw new Error('stream boom');
+}
+
+/**
+ * Yields once, then waits until `signal` aborts before completing — used to
+ * prove shutdownAndWait waits for tracked streams to finish after abort.
+ */
+async function* gateOnAbort(signal: AbortSignal): AsyncGenerator<string> {
+ yield 'started';
+ if (signal.aborted) {
+ return;
+ }
+ await new Promise<void>(resolve => {
+ signal.addEventListener('abort', () => resolve(), { once: true });
+ });
+}
+
+describe('ActiveTurnRegistry', () => {
+ it('track passes events through and removes the run when the stream completes', async () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ const tracked = registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: values([1, 2, 3]),
+ });
+
+ const seen: number[] = [];
+ for await (const value of tracked) {
+ seen.push(value);
+ }
+ assert.deepEqual(seen, [1, 2, 3]);
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ false,
+ );
+ });
+
+ it('track cleans up when the consumer breaks early', async () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ const tracked = registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: values([1, 2, 3]),
+ });
+
+ for await (const value of tracked) {
+ assert.equal(value, 1);
+ break;
+ }
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ false,
+ );
+ });
+
+ it('track cleans up when the stream throws', async () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ const tracked = registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: throwingStream(),
+ });
+
+ await assert.rejects(async () => {
+ for await (const value of tracked) {
+ void value;
+ }
+ }, /stream boom/);
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ false,
+ );
+ });
+
+ it('cancelIfRunning aborts with the given reason and returns true', () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ void registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: values([1]),
+ });
+
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ true,
+ );
+ assert.equal(abortController.signal.aborted, true);
+ assert.equal(abortController.signal.reason, CancellationReason.ClientCancelled);
+ });
+
+ it('cancelIfRunning returns false for unknown ids', () => {
+ const registry = new ActiveTurnRegistry();
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 'missing',
+ turnId: 'missing',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ false,
+ );
+ });
+
+ it('cancelIfRunning does not re-abort an already-aborted controller', () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ void registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: values([1]),
+ });
+ abortController.abort(CancellationReason.ClientCancelled);
+
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.Abandoned,
+ }),
+ true,
+ );
+ assert.equal(abortController.signal.reason, CancellationReason.ClientCancelled);
+ });
+
+ it('shutdownAndWait aborts runs and waits until tracked streams finish', async () => {
+ const registry = new ActiveTurnRegistry();
+ const abortController = new AbortController();
+ const tracked = registry.track({
+ sessionId: 's1',
+ turnId: 't1',
+ abortController,
+ stream: gateOnAbort(abortController.signal),
+ });
+
+ const drain = (async () => {
+ for await (const value of tracked) {
+ void value;
+ }
+ })();
+
+ await registry.shutdownAndWait(CancellationReason.Abandoned);
+ assert.equal(abortController.signal.aborted, true);
+ assert.equal(abortController.signal.reason, CancellationReason.Abandoned);
+ await drain;
+ assert.equal(
+ registry.cancelIfRunning({
+ sessionId: 's1',
+ turnId: 't1',
+ abortReason: CancellationReason.ClientCancelled,
+ }),
+ false,
+ );
+ });
+
+ it('late track after shutdownAndWait aborts immediately with the shutdown reason', async () => {
+ const registry = new ActiveTurnRegistry();
+ await registry.shutdownAndWait(CancellationReason.Abandoned);
+
+ const abortController = new AbortController();
+ const tracked = registry.track({
+ sessionId: 's1',
+ turnId: 'late',
+ abortController,
+ stream: values(['x']),
+ });
+
+ assert.equal(abortController.signal.aborted, true);
+ assert.equal(abortController.signal.reason, CancellationReason.Abandoned);
+
+ for await (const value of tracked) {
+ void value;
+ }
+ });
+});
diff --git a/packages/server/src/runtime/activeTurns.ts b/packages/server/src/runtime/activeTurns.ts
index 18d288e12..592348b55 100644
--- a/packages/server/src/runtime/activeTurns.ts
+++ b/packages/server/src/runtime/activeTurns.ts
@@ -1,34 +1,71 @@
/**
- * In-memory registry of turns running in this process, keyed by session. A
- * session may have any number of concurrent turns; the registry makes no
- * attempt to enforce one turn per session (that would not hold across
- * replicas anyway). Cancel code aborts runs through their AbortControllers;
+ * In-memory registry of turns executing in this process. Keys are
+ * `${sessionId}:${turnId}`. Cancel aborts via AbortController;
* TurnHandle.stream() writes the terminal state when the signal fires.
+ * `track()` owns registration and cleanup around the stream lifecycle.
*/
+import { CancellationReason } from '@truefoundry/utils/agent-session';
+
+interface ActiveTurnRun {
+ abortController: AbortController;
+ waitUntilCompleted: Promise<void>;
+ markCompleted: () => void;
+}
+
+function activeTurnKey(sessionId: string, turnId: string): string {
+ return `${sessionId}:${turnId}`;
+}
+
export class ActiveTurnRegistry {
- /** sessionId -> (turnId -> AbortController) */
- private readonly runs = new Map<string, Map<string, AbortController>>();
-
- /** Registers a running turn under its session. */
- register(input: { sessionId: string; turnId: string; abortController: AbortController }): void {
- let sessionRuns = this.runs.get(input.sessionId);
- if (!sessionRuns) {
- sessionRuns = new Map();
- this.runs.set(input.sessionId, sessionRuns);
- }
- sessionRuns.set(input.turnId, input.abortController);
- }
+ private readonly runs = new Map<string, ActiveTurnRun>();
+ private alreadyShutDownAbortReason: CancellationReason | undefined;
+
+ /**
+ * Registers the run immediately, then returns a generator that forwards
+ * `stream` and removes the run when the stream completes (or the consumer
+ * exits early). If shutdown has already begun, aborts the controller with
+ * the shutdown reason so the turn ends as abandoned.
+ */
+ track<T>(input: {
+ sessionId: string;
+ turnId: string;
+ abortController: AbortController;
+ stream: AsyncIterable<T>;
+ }): AsyncGenerator<T> {
+ const key = activeTurnKey(input.sessionId, input.turnId);
+ const { promise: waitUntilCompleted, resolve } = Promise.withResolvers<undefined>();
+ const markCompleted = (): void => {
+ resolve(undefined);
+ };
+ const run: ActiveTurnRun = {
+ abortController: input.abortController,
+ waitUntilCompleted,
+ markCompleted,
+ };
+ this.runs.set(key, run);
- /** Removes the entry once the turn's stream has fully drained. */
- finish(input: { sessionId: string; turnId: string }): void {
- const sessionRuns = this.runs.get(input.sessionId);
- if (!sessionRuns) {
- return;
+ // Accepted HTTP may still call track() after shutdownAndWait snapshotted;
+ // abort immediately so the turn ends as abandoned instead of running past exit.
+ if (this.alreadyShutDownAbortReason !== undefined) {
+ if (!input.abortController.signal.aborted) {
+ input.abortController.abort(this.alreadyShutDownAbortReason);
+ }
}
- sessionRuns.delete(input.turnId);
- if (sessionRuns.size === 0) {
- this.runs.delete(input.sessionId);
+
+ const complete = () => {
+ run.markCompleted();
+ if (this.runs.get(key) === run) {
+ this.runs.delete(key);
+ }
+ };
+ async function* tracked(): AsyncGenerator<T> {
+ try {
+ yield* input.stream;
+ } finally {
+ complete();
+ }
}
+ return tracked();
}
/**
@@ -37,14 +74,33 @@ export class ActiveTurnRegistry {
* turn that is not running is a no-op, mirroring the store's
* first-terminal-write-wins rule.
*/
- cancelIfRunning(input: { sessionId: string; turnId: string; abortReason?: string }): boolean {
- const abortController = this.runs.get(input.sessionId)?.get(input.turnId);
- if (!abortController) {
+ cancelIfRunning(input: { sessionId: string; turnId: string; abortReason: CancellationReason }): boolean {
+ const run = this.runs.get(activeTurnKey(input.sessionId, input.turnId));
+ if (!run) {
return false;
}
- if (!abortController.signal.aborted) {
- abortController.abort(input.abortReason);
+ if (!run.abortController.signal.aborted) {
+ run.abortController.abort(input.abortReason);
}
return true;
}
+
+ /**
+ * Enter shutdown mode, abort every run with `abortReason`, and wait until the
+ * registry is empty. Late `track()` calls (in-flight HTTP that registered after
+ * the first snapshot) are aborted immediately as `abortReason` and included in
+ * subsequent wait iterations.
+ */
+ async shutdownAndWait(abortReason: CancellationReason): Promise<void> {
+ this.alreadyShutDownAbortReason = abortReason;
+ while (this.runs.size > 0) {
+ const pending = Array.from(this.runs.values());
+ for (const run of pending) {
+ if (!run.abortController.signal.aborted) {
+ run.abortController.abort(abortReason);
+ }
+ }
+ await Promise.allSettled(pending.map(run => run.waitUntilCompleted));
+ }
+ }
}