Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

Cancellation fails after a restart in distributed mode (#294)

Heer Ambavi5b981abc70f9
diff --git a/.changeset/freeze-running-turn-when-cancel-cannot-abort.md b/.changeset/freeze-running-turn-when-cancel-cannot-abort.md
new file mode 100644
index 000000000..71a93f75a
--- /dev/null
+++ b/.changeset/freeze-running-turn-when-cancel-cannot-abort.md
@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge-core': patch
+'@truefoundry/trueforge': patch
+---
+
+Cancel a session even when the owning executor is gone (restart) or Redis cannot confirm abort. Freeze the running turn in the store so a new turn can start. `freezeAndGetTurn` now takes the cancellation reason (barge-in stays `cancelled-for-next-turn`; explicit cancel stays `client-cancelled`). Redis timeout and transport failures still freeze, with a warning that the cancel is not clean.
diff --git a/packages/trueforge-core/src/agent-session/SessionHandle.ts b/packages/trueforge-core/src/agent-session/SessionHandle.ts
index 2a7a204c3..e116cb98b 100644
--- a/packages/trueforge-core/src/agent-session/SessionHandle.ts
+++ b/packages/trueforge-core/src/agent-session/SessionHandle.ts
@@ -90,14 +90,14 @@ function collectContextAppends(
})();
}
-function cancelledTurnDoneEvent(): TurnDoneEvent {
+function cancelledTurnDoneEvent(reason: CancellationReason): TurnDoneEvent {
return {
type: EventType.TURN_DONE,
id: newEventId(),
created_at: new Date().toISOString(),
state: {
status: 'cancelled',
- reason: CancellationReason.CancelledForNextTurn,
+ reason,
completed_at: new Date().toISOString(),
},
thread_id: null,
@@ -179,10 +179,9 @@ export class SessionHandle<
}): Promise<TurnHandle<TTurnCustom>> {
const previousTurnId = resolvePreviousTurnId(input.previous_turn_id, this.session.last_turn_id);
const previous = previousTurnId
- ? await this.store.freezeAndGetTurn({
- session_id: this.session.session_id,
+ ? await this.freezeTurn({
turn_id: previousTurnId,
- turn_done_event: cancelledTurnDoneEvent(),
+ reason: CancellationReason.CancelledForNextTurn,
})
: undefined;
@@ -309,6 +308,20 @@ export class SessionHandle<
}
}
+ /**
+ * Conditionally cancel a running turn in this session and return the
+ * now-immutable record. Already-terminal turns are returned unchanged.
+ * Missing turn → {@link TurnNotFoundError}.
+ */
+ async freezeTurn(input: { turn_id: string; reason: CancellationReason }): Promise<TurnRecord<TTurnCustom>> {
+ return this.store.freezeAndGetTurn({
+ session_id: this.session.session_id,
+ turn_id: input.turn_id,
+ reason: input.reason,
+ turn_done_event: cancelledTurnDoneEvent(input.reason),
+ });
+ }
+
/** Returns the turn handle (store-backed; not executable), or undefined if not found. */
async getTurn(turn_id: string): Promise<TurnHandle<TTurnCustom> | undefined> {
return TurnHandle.get({
diff --git a/packages/trueforge-core/src/agent-session/store/ISessionStore.ts b/packages/trueforge-core/src/agent-session/store/ISessionStore.ts
index 17eab046a..30cfa6bf6 100644
--- a/packages/trueforge-core/src/agent-session/store/ISessionStore.ts
+++ b/packages/trueforge-core/src/agent-session/store/ISessionStore.ts
@@ -11,7 +11,7 @@ import type { SessionRecord } from '../models/SessionRecord';
import type { TurnRecord } from '../models/TurnRecord';
import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';
import type { TokenPagination } from '../schemas/pagination';
-import type { TerminalTurnState } from '../schemas/turn';
+import type { CancellationReason, TerminalTurnState } from '../schemas/turn';
/**
* Caller-supplied fields for creating a session; the store owns timestamps and tip state.
@@ -107,6 +107,8 @@ export interface CreateTurnInput<TTurnCustom extends object = Record<string, nev
export interface FreezeAndGetTurnInput {
session_id: string;
turn_id: string;
+ /** Written onto the turn row when the freeze actually cancels a running turn. */
+ reason: CancellationReason;
/** Caller-built cancelled turn.done event; store persists atomically with state flip when it cancels. */
turn_done_event: PersistedTurnEvent;
}
@@ -279,11 +281,10 @@ export interface ISessionStore<
createTurn(input: CreateTurnInput<TTurnCustom>): Promise<void>;
/**
- * One tx: (a) conditionally cancel a running turn with
- * {@link CancellationReason.CancelledForNextTurn}; (b) if it did cancel,
- * insert the caller-built `turn_done_event` — already-terminal turns skip the
- * event insert; (c) return the now-immutable turn record. Missing turn →
- * {@link TurnNotFoundError}.
+ * One tx: (a) conditionally cancel a running turn with `reason`; (b) if it
+ * did cancel, insert the caller-built `turn_done_event` — already-terminal
+ * turns skip the event insert; (c) return the now-immutable turn record.
+ * Missing turn → {@link TurnNotFoundError}.
*/
freezeAndGetTurn(input: FreezeAndGetTurnInput): Promise<TurnRecord<TTurnCustom>>;
diff --git a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts
index 2e36eb560..fb477beb8 100644
--- a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts
+++ b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts
@@ -4,7 +4,7 @@ import type { SessionRecord } from '../models/SessionRecord';
import type { TurnRecord, TurnSnapshot } from '../models/TurnRecord';
import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';
import type { TokenPagination } from '../schemas/pagination';
-import { CancellationReason, type TerminalTurnState } from '../schemas/turn';
+import type { TerminalTurnState } from '../schemas/turn';
import { assertCreateTurnThreadDelta } from './assertCreateTurnThreadDelta';
import type {
AddThreadsInput,
@@ -346,7 +346,7 @@ export class InMemorySessionStore<
if (turn.state.status === 'running') {
const cancelledState: TerminalTurnState = {
status: 'cancelled',
- reason: CancellationReason.CancelledForNextTurn,
+ reason: input.reason,
completed_at: new Date().toISOString(),
};
turn.state = cancelledState;
diff --git a/packages/trueforge-core/tests/agent-session/sessions.test.ts b/packages/trueforge-core/tests/agent-session/sessions.test.ts
index f9a53bd7a..3c37ec1fc 100644
--- a/packages/trueforge-core/tests/agent-session/sessions.test.ts
+++ b/packages/trueforge-core/tests/agent-session/sessions.test.ts
@@ -1,6 +1,8 @@
import { EventType } from '../../src/agent-session/schemas/events';
+import { CancellationReason } from '../../src/agent-session/schemas/turn';
import { Sessions } from '../../src/agent-session/Sessions';
import { InMemorySessionStore } from '../../src/agent-session/store/InMemorySessionStore';
+import { TurnNotFoundError } from '../../src/agent-session/store/SessionStoreErrors';
import { TurnHandle } from '../../src/agent-session/TurnHandle';
import { makeAgentSpec, makeTestResolver, mintTestTurnId } from './testHelpers';
@@ -98,6 +100,34 @@ describe('Sessions / SessionHandle / TurnHandle (storage + createTurn)', () => {
expect(getSession).not.toHaveBeenCalled();
});
+ it('freezeTurn cancels a running turn with the given reason', async () => {
+ const store = new InMemorySessionStore();
+ const sessions = new Sessions({ sessionStore: store });
+ const session = await sessions.create({
+ tenant_id: tenant,
+ session_id: 's1',
+ created_by: 'user-1',
+ agent: { type: 'inline', spec: makeAgentSpec() },
+ });
+ const turn = await session.createTurn({
+ turn_id: mintTestTurnId(),
+ previous_turn_id: 'none',
+ signal: new AbortController().signal,
+ resolver: makeTestResolver(),
+ });
+ const frozen = await session.freezeTurn({
+ turn_id: turn.id,
+ reason: CancellationReason.ClientCancelled,
+ });
+ expect(frozen.state).toMatchObject({
+ status: 'cancelled',
+ reason: CancellationReason.ClientCancelled,
+ });
+ await expect(
+ session.freezeTurn({ turn_id: 'missing-turn', reason: CancellationReason.ClientCancelled }),
+ ).rejects.toBeInstanceOf(TurnNotFoundError);
+ });
+
it('custom value vs merge-fn', async () => {
const store = new InMemorySessionStore<{ tag: string }, { n: number }>();
const sessions = new Sessions({ sessionStore: store });
diff --git a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts
index 5edb06e5b..f67685ae0 100644
--- a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts
+++ b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts
@@ -97,6 +97,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
() =>
store.freezeAndGetTurn({
...keys,
+ reason: CancellationReason.ClientCancelled,
turn_done_event: makeTurnDoneEvent(makeCancelledTurnState(CancellationReason.ClientCancelled)),
}),
() =>
@@ -1220,9 +1221,11 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
const record = await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
+ reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
expect(record.state.status).toBe('cancelled');
+ expect(record.state).toMatchObject({ reason: CancellationReason.CancelledForNextTurn });
const { data } = await store.listTurnEvents({
session_id: sessionId,
turn_id: 'turn-1',
@@ -1297,6 +1300,23 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
}
});
+ it('cancels a running turn with the caller-supplied reason', async () => {
+ const store = createStore();
+ await seedSession(store);
+ await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
+ const cancelledState = makeCancelledTurnState(CancellationReason.ClientCancelled);
+ const record = await store.freezeAndGetTurn({
+ session_id: sessionId,
+ turn_id: 'turn-1',
+ reason: CancellationReason.ClientCancelled,
+ turn_done_event: makeTurnDoneEvent(cancelledState),
+ });
+ expect(record.state).toMatchObject({
+ status: 'cancelled',
+ reason: CancellationReason.ClientCancelled,
+ });
+ });
+
it('on an already-terminal turn is a plain read without duplicating turn.done', async () => {
const store = createStore();
await seedSession(store);
@@ -1306,6 +1326,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
+ reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(doneState),
});
const { data } = await store.listTurnEvents({
@@ -1327,11 +1348,13 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
+ reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
}),
store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
+ reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
}),
]);
@@ -1358,6 +1381,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
+ reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
}),
store.updateTurnState({
diff --git a/packages/trueforge/src/apis/sessions.ts b/packages/trueforge/src/apis/sessions.ts
index 5078af7f0..eb92d3657 100644
--- a/packages/trueforge/src/apis/sessions.ts
+++ b/packages/trueforge/src/apis/sessions.ts
@@ -2,21 +2,24 @@
* DB-backed sessions API (mounted at /api/v1/sessions).
*/
import { OpenAPIHono, type RouteHandler } from '@hono/zod-openapi';
-import type { ISessionStore, SessionRecord, Sessions } from '@truefoundry/trueforge-core/agent-session';
+import type { ISessionStore, SessionHandle, SessionRecord, Sessions } from '@truefoundry/trueforge-core/agent-session';
import {
CancellationReason,
SessionStoreConflictError,
SessionStoreInvariantError,
SessionStoreNotFoundError,
+ TurnNotFoundError,
} from '@truefoundry/trueforge-core/agent-session';
-import type {
- RouteHandler as RequestReplyRouteHandler,
- RequestReplyRouter,
+import { extractErrorLogFields } from '@truefoundry/trueforge-core/core';
+import {
+ redisRequest,
+ RequestTimeoutError,
+ type RouteHandler as RequestReplyRouteHandler,
+ type RequestReplyRouter,
} from '@truefoundry/trueforge-core/request-reply';
-import { NoResponderError, redisRequest, RequestTimeoutError } from '@truefoundry/trueforge-core/request-reply';
-import { HTTPException } from 'hono/http-exception';
import type { RedisClientType } from 'redis';
import { ulid } from 'ulid';
+import type { Logger } from 'winston';
import { z } from 'zod';
import type { ResolveUserContext } from '../auth/identity';
import configuration from '../config';
@@ -76,6 +79,7 @@ export interface SessionsRouterDeps {
redis?: RedisClientType | undefined;
requestReplyRouter: RequestReplyRouter;
resolveUserContext: ResolveUserContext;
+ logger: Logger;
}
function cancelTurnOnThisExecutor(
@@ -112,23 +116,33 @@ export function cancelSessionTurnPeerHandler(activeTurns: ActiveTurnRegistry): R
};
}
-/** A registry to abort in, durable state to read, and a way to reach peers. */
+/** A registry to abort in, a session to freeze, durable state to read, and a way to reach peers. */
export interface CancelTurnDeps {
activeTurns: ActiveTurnRegistry;
+ session: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;
sessionStore: Pick<ISessionStore, 'getTurn'>;
redis?: RedisClientType | undefined;
+ logger: Pick<Logger, 'warn'>;
}
/**
* Cancels the turn wherever it runs: locally or on the owning peer over Redis
* request-reply. Callers state the motive; default is a plain client cancel.
- * Owner failures throw HTTPException (412 unreachable, 424 timed out).
+ *
+ * A confirmed abort (this process, or peer HTTP 200) lets TurnHandle persist
+ * the terminal state. If abort cannot be confirmed, this replica freezes the
+ * turn in the store so the session is not stuck `running`.
+ *
+ * Redis timeout and Redis/transport failures are not a clean cancellation —
+ * the owning replica may still be executing — but the turn is still frozen.
+ * Later writes from that replica lose to first-terminal-write-wins.
*/
export async function cancelSessionTurn(
deps: CancelTurnDeps,
- input: { sessionId: string; turnId: string; reason?: CancellationReason },
+ input: { turnId: string; reason?: CancellationReason },
): Promise<void> {
- const { sessionId, turnId, reason = CancellationReason.ClientCancelled } = input;
+ const { turnId, reason = CancellationReason.ClientCancelled } = input;
+ const sessionId = deps.session.session_id;
const turn = await deps.sessionStore.getTurn({
session_id: sessionId,
@@ -141,7 +155,7 @@ export async function cancelSessionTurn(
const owner = executorFromTurnId(turnId);
// Without a Redis client there is no peer to ask, so an id naming another
- // replica falls through to the local lookup and finds nothing.
+ // replica falls through to the local lookup and freezes if the run is gone.
if (owner !== configuration.EXECUTOR_ID && deps.redis) {
try {
const reply = await redisRequest<CancelPeerBody>({
@@ -156,34 +170,45 @@ export async function cancelSessionTurn(
pollIntervalMs: configuration.REDIS_REQUEST_REPLY_POLL_INTERVAL_MS,
},
});
- if (reply.status !== 200 && reply.status !== 412) {
- throw new HTTPException(500, { message: 'Failed to cancel turn on the owning executor', cause: reply });
+ if (reply.status === 200) {
+ return;
}
} catch (error) {
- if (error instanceof NoResponderError) {
- throw new HTTPException(412, {
- message: `Executor owning the running turn is unreachable: ${owner}`,
- cause: error,
- });
- }
+ const fields = {
+ sessionId,
+ turnId,
+ owner,
+ ...extractErrorLogFields(error),
+ };
if (error instanceof RequestTimeoutError) {
- throw new HTTPException(424, {
- message: 'Timed out waiting for the owning executor to cancel the turn',
- cause: error,
- });
+ deps.logger.warn('Timed out waiting for owning executor to cancel; freezing the running turn', fields);
+ } else {
+ deps.logger.warn('Failed to reach owning executor over Redis; freezing the running turn', fields);
}
- if (error instanceof HTTPException) {
- throw error;
- }
- throw new HTTPException(500, {
- message: `Unexpected error while cancelling the turn on the owning executor: ${owner}`,
- cause: error,
- });
}
+ await freezeTurnIgnoringMissing(deps.session, { turnId, reason });
return;
}
- cancelTurnOnThisExecutor(deps.activeTurns, { sessionId, turnId, reason });
+ const aborted = cancelTurnOnThisExecutor(deps.activeTurns, { sessionId, turnId, reason });
+ if (!aborted) {
+ await freezeTurnIgnoringMissing(deps.session, { turnId, reason });
+ }
+}
+
+/** Freeze a running turn; missing turns are a no-op (already gone). */
+async function freezeTurnIgnoringMissing(
+ session: Pick<SessionHandle, 'freezeTurn'>,
+ input: { turnId: string; reason: CancellationReason },
+): Promise<void> {
+ try {
+ await session.freezeTurn({ turn_id: input.turnId, reason: input.reason });
+ } catch (error) {
+ if (error instanceof TurnNotFoundError) {
+ return;
+ }
+ throw error;
+ }
}
const FORBIDDEN_SESSION_ACCESS = 'Only the session creator can access this session';
@@ -327,19 +352,19 @@ export function createSessionsRouter(deps: SessionsRouterDeps) {
const cancelSessionHandler: RouteHandler<typeof cancelSessionRoute> = async c => {
const { session_id: sessionId } = c.req.valid('param');
- const record = await deps.sessionStore.getSession({ tenant_id: TENANT_ID, session_id: sessionId });
- if (!record) {
+ const session = await deps.sessions.get({ tenant_id: TENANT_ID, session_id: sessionId });
+ if (!session) {
return c.json({ error: { message: `Session not found: ${sessionId}` } }, 404);
}
- if (!checkSessionAccess({ userRef: deps.resolveUserContext(c).userRef, createdBy: record.created_by })) {
+ if (!checkSessionAccess({ userRef: deps.resolveUserContext(c).userRef, createdBy: session.record.created_by })) {
return c.json({ error: { message: FORBIDDEN_SESSION_ACCESS } }, 403);
}
- const turnId = record.last_turn_id;
+ const turnId = session.record.last_turn_id;
if (!turnId) {
return c.json({}, 200);
}
- await cancelSessionTurn(deps, { sessionId, turnId });
+ await cancelSessionTurn({ ...deps, session }, { turnId });
return c.json({}, 200);
};
diff --git a/packages/trueforge/src/app.ts b/packages/trueforge/src/app.ts
index 74994d02b..ee32e5fb4 100644
--- a/packages/trueforge/src/app.ts
+++ b/packages/trueforge/src/app.ts
@@ -272,6 +272,7 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {
redis: deps.redis,
requestReplyRouter: deps.requestReplyRouter,
resolveUserContext: resolveUserContext,
+ logger: deps.logger,
}),
),
);
diff --git a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts b/packages/trueforge/src/db/postgres/session-store/queries/turns.ts
index 6e5827f2d..f89176048 100644
--- a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts
+++ b/packages/trueforge/src/db/postgres/session-store/queries/turns.ts
@@ -1,6 +1,5 @@
import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';
import {
- CancellationReason,
type TerminalTurnState,
type TurnInputItem,
type TurnState,
@@ -604,7 +603,7 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
return await db.transaction().execute(async trx => {
const cancelledState: TerminalTurnState = {
status: 'cancelled',
- reason: CancellationReason.CancelledForNextTurn,
+ reason: input.reason,
completed_at: new Date().toISOString(),
};
diff --git a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts b/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts
index ca9a346a0..0c5416127 100644
--- a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts
+++ b/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts
@@ -1,6 +1,5 @@
import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';
import {
- CancellationReason,
type TerminalTurnState,
type TurnInputItem,
type TurnState,
@@ -657,7 +656,7 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
return await db.transaction().execute(async trx => {
const cancelledState: TerminalTurnState = {
status: 'cancelled',
- reason: CancellationReason.CancelledForNextTurn,
+ reason: input.reason,
completed_at: nowIso(),
};
diff --git a/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts b/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts
index c931be0ad..832d68ca5 100644
--- a/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts
+++ b/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts
@@ -48,6 +48,7 @@ describe('public CRUD after session deletion', () => {
redis: createClient(),
requestReplyRouter: new RequestReplyRouter(),
resolveUserContext: () => LOCAL_USER_CONTEXT,
+ logger: createLogger({ silent: true }),
}),
);
app.route(
diff --git a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts b/packages/trueforge/tests/unit/apis/sessionHttp.test.ts
index 5b44fd816..dd40b0027 100644
--- a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts
+++ b/packages/trueforge/tests/unit/apis/sessionHttp.test.ts
@@ -2,6 +2,7 @@ import { OpenAPIHono } from '@hono/zod-openapi';
import { AgentSpecSchema, Sessions } from '@truefoundry/trueforge-core/agent-session';
import { RequestReplyRouter } from '@truefoundry/trueforge-core/request-reply';
import { createClient } from 'redis';
+import { createLogger } from 'winston';
import { createSessionsRouter, TENANT_ID } from '../../../src/apis/sessions';
import { LOCAL_USER_CONTEXT } from '../../../src/auth/identity';
import { migrateSqliteToLatest } from '../../../src/db/migrateSqlite';
@@ -75,6 +76,7 @@ describe('sessions HTTP agent binding', () => {
redis: createClient(),
requestReplyRouter: new RequestReplyRouter(),
resolveUserContext: () => LOCAL_USER_CONTEXT,
+ logger: createLogger({ silent: true }),
}),
);
});
diff --git a/packages/trueforge/tests/unit/apis/sessions.test.ts b/packages/trueforge/tests/unit/apis/sessions.test.ts
index 72bba1551..7af166b49 100644
--- a/packages/trueforge/tests/unit/apis/sessions.test.ts
+++ b/packages/trueforge/tests/unit/apis/sessions.test.ts
@@ -1,13 +1,30 @@
-import type { ISessionStore, TurnRecord, TurnState } from '@truefoundry/trueforge-core/agent-session';
-import { CancellationReason } from '@truefoundry/trueforge-core/agent-session';
-import { HTTPException } from 'hono/http-exception';
-import { createClient, type RedisClientType } from 'redis';
+import type { ISessionStore, SessionHandle, TurnRecord, TurnState } from '@truefoundry/trueforge-core/agent-session';
+import { CancellationReason, TurnNotFoundError } from '@truefoundry/trueforge-core/agent-session';
+import { NoResponderError, redisRequest, RequestTimeoutError } from '@truefoundry/trueforge-core/request-reply';
+import type { RedisClientType } from 'redis';
import { cancelSessionTurn } from '../../../src/apis/sessions';
import configuration from '../../../src/config';
import { ActiveTurnRegistry } from '../../../src/runtime/activeTurns';
import { mintPeeredTurnId } from '../../../src/runtime/peeringIds';
+jest.mock('@truefoundry/trueforge-core/request-reply', () => {
+ const actual = jest.requireActual<typeof import('@truefoundry/trueforge-core/request-reply')>(
+ '@truefoundry/trueforge-core/request-reply',
+ );
+ return {
+ ...actual,
+ redisRequest: jest.fn(),
+ };
+});
+
+const redisRequestMock = jest.mocked(redisRequest);
+
const SESSION_ID = 's1';
+const REDIS = {} as RedisClientType;
+
+function silentLogger(): { warn: jest.Mock } {
+ return { warn: jest.fn() };
+}
function turnRecord(turnId: string, state: TurnState): TurnRecord {
return {
@@ -25,15 +42,37 @@ function turnRecord(turnId: string, state: TurnState): TurnRecord {
};
}
-/** Only `getTurn` is exercised, which is all CancelTurnDeps asks for. */
function storeReturning(turn: TurnRecord | undefined): Pick<ISessionStore, 'getTurn'> {
return { getTurn: () => Promise.resolve(turn) };
}
-/** Unconnected, so any attempt to reach a peer fails loudly instead of silently passing. */
-function unconnectedRedis(): RedisClientType {
- const client: RedisClientType = createClient();
- return client;
+function sessionHandle(): Pick<SessionHandle, 'session_id' | 'freezeTurn'> {
+ return {
+ session_id: SESSION_ID,
+ freezeTurn: jest.fn().mockResolvedValue(undefined),
+ };
+}
+
+function cancelDeps(input: {
+ activeTurns: ActiveTurnRegistry;
+ turn: TurnRecord | undefined;
+ session?: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;
+ redis?: RedisClientType;
+ logger?: { warn: jest.Mock };
+}): {
+ activeTurns: ActiveTurnRegistry;
+ session: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;
+ sessionStore: Pick<ISessionStore, 'getTurn'>;
+ redis?: RedisClientType;
+ logger: { warn: jest.Mock };
+} {
+ return {
+ activeTurns: input.activeTurns,
+ session: input.session ?? sessionHandle(),
+ sessionStore: storeReturning(input.turn),
+ ...(input.redis === undefined ? {} : { redis: input.redis }),
+ logger: input.logger ?? silentLogger(),
+ };
}
/** Registers a live run without consuming its stream, mirroring a turn mid-execution. */
@@ -52,53 +91,154 @@ function trackRun(registry: ActiveTurnRegistry, turnId: string): AbortController
}
describe('cancelSessionTurn', () => {
- it('aborts a turn running in this process', async () => {
+ beforeEach(() => {
+ redisRequestMock.mockReset();
+ });
+
+ it('aborts a turn running in this process without freezing', async () => {
const activeTurns = new ActiveTurnRegistry();
const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);
const abortController = trackRun(activeTurns, turnId);
+ const session = sessionHandle();
- await cancelSessionTurn(
- {
- activeTurns,
- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),
- },
- { sessionId: SESSION_ID, turnId },
- );
+ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {
+ turnId,
+ });
expect(abortController.signal.aborted).toBe(true);
expect(abortController.signal.reason).toBe(CancellationReason.ClientCancelled);
+ expect(session.freezeTurn).not.toHaveBeenCalled();
+ });
+
+ it('freezes when this executor owns the turn id but the run is gone', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);
+ const session = sessionHandle();
+
+ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {
+ turnId,
+ });
+
+ expect(session.freezeTurn).toHaveBeenCalledWith({
+ turn_id: turnId,
+ reason: CancellationReason.ClientCancelled,
+ });
+ });
+
+ it('freezes when the run is not in this process and there is no Redis client', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+
+ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {
+ turnId,
+ });
+
+ expect(session.freezeTurn).toHaveBeenCalledWith({
+ turn_id: turnId,
+ reason: CancellationReason.ClientCancelled,
+ });
+ });
+
+ it('does not freeze when the owning peer aborts', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+ redisRequestMock.mockResolvedValue({ status: 200, body: {} });
+
+ await cancelSessionTurn(
+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS }),
+ { turnId },
+ );
+
+ expect(redisRequestMock).toHaveBeenCalled();
+ expect(session.freezeTurn).not.toHaveBeenCalled();
});
- it('does not look for a peer without a Redis client', async () => {
+ it('freezes when the owning peer is alive but the turn is not running there', async () => {
const activeTurns = new ActiveTurnRegistry();
const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+ redisRequestMock.mockResolvedValue({ status: 412, body: { message: 'Turn is not running on this executor' } });
+
+ await cancelSessionTurn(
+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS }),
+ { turnId },
+ );
+
+ expect(session.freezeTurn).toHaveBeenCalled();
+ });
+
+ it('freezes when the owning executor is unreachable', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+ const logger = silentLogger();
+ redisRequestMock.mockRejectedValue(new NoResponderError('other1'));
- // A peer hop here would dereference the absent client and throw.
await expect(
cancelSessionTurn(
- {
- activeTurns,
- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),
- },
- { sessionId: SESSION_ID, turnId },
+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),
+ { turnId },
),
).resolves.toBeUndefined();
+ expect(session.freezeTurn).toHaveBeenCalled();
+ expect(logger.warn).toHaveBeenCalledWith(
+ 'Failed to reach owning executor over Redis; freezing the running turn',
+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1' }),
+ );
});
- it('routes to the owning peer when the turn id names another replica', async () => {
+ it('freezes when waiting for the owning executor times out', async () => {
const activeTurns = new ActiveTurnRegistry();
const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+ const logger = silentLogger();
+ redisRequestMock.mockRejectedValue(new RequestTimeoutError(60_000));
- // The unconnected client makes the hop fail; the point is that it happened.
await expect(
cancelSessionTurn(
- {
- activeTurns,
- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),
- redis: unconnectedRedis(),
- },
- { sessionId: SESSION_ID, turnId },
+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),
+ { turnId },
),
- ).rejects.toBeInstanceOf(HTTPException);
+ ).resolves.toBeUndefined();
+ expect(session.freezeTurn).toHaveBeenCalled();
+ expect(logger.warn).toHaveBeenCalledWith(
+ 'Timed out waiting for owning executor to cancel; freezing the running turn',
+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1' }),
+ );
+ });
+
+ it('freezes when Redis itself cannot be reached', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId('other1');
+ const session = sessionHandle();
+ const logger = silentLogger();
+ redisRequestMock.mockRejectedValue(new Error('Redis connection closed'));
+
+ await expect(
+ cancelSessionTurn(
+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),
+ { turnId },
+ ),
+ ).resolves.toBeUndefined();
+ expect(session.freezeTurn).toHaveBeenCalled();
+ expect(logger.warn).toHaveBeenCalledWith(
+ 'Failed to reach owning executor over Redis; freezing the running turn',
+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1', error: 'Redis connection closed' }),
+ );
+ });
+
+ it('treats a missing turn as a successful cancel', async () => {
+ const activeTurns = new ActiveTurnRegistry();
+ const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);
+ const session = sessionHandle();
+ session.freezeTurn = jest.fn().mockRejectedValue(new TurnNotFoundError(turnId));
+
+ await expect(
+ cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {
+ turnId,
+ }),
+ ).resolves.toBeUndefined();
});
});