The open-source agent harness - the runtime layer that turns an LLM into a working agent.
Cancellation fails after a restart in distributed mode (#294)
Heer Ambavi5b981abc70f9
diff --git a/.changeset/freeze-running-turn-when-cancel-cannot-abort.md b/.changeset/freeze-running-turn-when-cancel-cannot-abort.mdnew file mode 100644index 000000000..71a93f75a--- /dev/null+++ b/.changeset/freeze-running-turn-when-cancel-cannot-abort.md@@ -0,0 +1,6 @@+---+'@truefoundry/trueforge-core': patch+'@truefoundry/trueforge': patch+---++Cancel a session even when the owning executor is gone (restart) or Redis cannot confirm abort. Freeze the running turn in the store so a new turn can start. `freezeAndGetTurn` now takes the cancellation reason (barge-in stays `cancelled-for-next-turn`; explicit cancel stays `client-cancelled`). Redis timeout and transport failures still freeze, with a warning that the cancel is not clean.diff --git a/packages/trueforge-core/src/agent-session/SessionHandle.ts b/packages/trueforge-core/src/agent-session/SessionHandle.tsindex 2a7a204c3..e116cb98b 100644--- a/packages/trueforge-core/src/agent-session/SessionHandle.ts+++ b/packages/trueforge-core/src/agent-session/SessionHandle.ts@@ -90,14 +90,14 @@ function collectContextAppends(})();}-function cancelledTurnDoneEvent(): TurnDoneEvent {+function cancelledTurnDoneEvent(reason: CancellationReason): TurnDoneEvent {return {type: EventType.TURN_DONE,id: newEventId(),created_at: new Date().toISOString(),state: {status: 'cancelled',- reason: CancellationReason.CancelledForNextTurn,+ reason,completed_at: new Date().toISOString(),},thread_id: null,@@ -179,10 +179,9 @@ export class SessionHandle<}): Promise<TurnHandle<TTurnCustom>> {const previousTurnId = resolvePreviousTurnId(input.previous_turn_id, this.session.last_turn_id);const previous = previousTurnId- ? await this.store.freezeAndGetTurn({- session_id: this.session.session_id,+ ? await this.freezeTurn({turn_id: previousTurnId,- turn_done_event: cancelledTurnDoneEvent(),+ reason: CancellationReason.CancelledForNextTurn,}): undefined;@@ -309,6 +308,20 @@ export class SessionHandle<}}+ /**+ * Conditionally cancel a running turn in this session and return the+ * now-immutable record. Already-terminal turns are returned unchanged.+ * Missing turn → {@link TurnNotFoundError}.+ */+ async freezeTurn(input: { turn_id: string; reason: CancellationReason }): Promise<TurnRecord<TTurnCustom>> {+ return this.store.freezeAndGetTurn({+ session_id: this.session.session_id,+ turn_id: input.turn_id,+ reason: input.reason,+ turn_done_event: cancelledTurnDoneEvent(input.reason),+ });+ }+/** Returns the turn handle (store-backed; not executable), or undefined if not found. */async getTurn(turn_id: string): Promise<TurnHandle<TTurnCustom> | undefined> {return TurnHandle.get({diff --git a/packages/trueforge-core/src/agent-session/store/ISessionStore.ts b/packages/trueforge-core/src/agent-session/store/ISessionStore.tsindex 17eab046a..30cfa6bf6 100644--- a/packages/trueforge-core/src/agent-session/store/ISessionStore.ts+++ b/packages/trueforge-core/src/agent-session/store/ISessionStore.ts@@ -11,7 +11,7 @@ import type { SessionRecord } from '../models/SessionRecord';import type { TurnRecord } from '../models/TurnRecord';import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';import type { TokenPagination } from '../schemas/pagination';-import type { TerminalTurnState } from '../schemas/turn';+import type { CancellationReason, TerminalTurnState } from '../schemas/turn';/*** Caller-supplied fields for creating a session; the store owns timestamps and tip state.@@ -107,6 +107,8 @@ export interface CreateTurnInput<TTurnCustom extends object = Record<string, nevexport interface FreezeAndGetTurnInput {session_id: string;turn_id: string;+ /** Written onto the turn row when the freeze actually cancels a running turn. */+ reason: CancellationReason;/** Caller-built cancelled turn.done event; store persists atomically with state flip when it cancels. */turn_done_event: PersistedTurnEvent;}@@ -279,11 +281,10 @@ export interface ISessionStore<createTurn(input: CreateTurnInput<TTurnCustom>): Promise<void>;/**- * One tx: (a) conditionally cancel a running turn with- * {@link CancellationReason.CancelledForNextTurn}; (b) if it did cancel,- * insert the caller-built `turn_done_event` — already-terminal turns skip the- * event insert; (c) return the now-immutable turn record. Missing turn →- * {@link TurnNotFoundError}.+ * One tx: (a) conditionally cancel a running turn with `reason`; (b) if it+ * did cancel, insert the caller-built `turn_done_event` — already-terminal+ * turns skip the event insert; (c) return the now-immutable turn record.+ * Missing turn → {@link TurnNotFoundError}.*/freezeAndGetTurn(input: FreezeAndGetTurnInput): Promise<TurnRecord<TTurnCustom>>;diff --git a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.tsindex 2e36eb560..fb477beb8 100644--- a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts+++ b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts@@ -4,7 +4,7 @@ import type { SessionRecord } from '../models/SessionRecord';import type { TurnRecord, TurnSnapshot } from '../models/TurnRecord';import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';import type { TokenPagination } from '../schemas/pagination';-import { CancellationReason, type TerminalTurnState } from '../schemas/turn';+import type { TerminalTurnState } from '../schemas/turn';import { assertCreateTurnThreadDelta } from './assertCreateTurnThreadDelta';import type {AddThreadsInput,@@ -346,7 +346,7 @@ export class InMemorySessionStore<if (turn.state.status === 'running') {const cancelledState: TerminalTurnState = {status: 'cancelled',- reason: CancellationReason.CancelledForNextTurn,+ reason: input.reason,completed_at: new Date().toISOString(),};turn.state = cancelledState;diff --git a/packages/trueforge-core/tests/agent-session/sessions.test.ts b/packages/trueforge-core/tests/agent-session/sessions.test.tsindex f9a53bd7a..3c37ec1fc 100644--- a/packages/trueforge-core/tests/agent-session/sessions.test.ts+++ b/packages/trueforge-core/tests/agent-session/sessions.test.ts@@ -1,6 +1,8 @@import { EventType } from '../../src/agent-session/schemas/events';+import { CancellationReason } from '../../src/agent-session/schemas/turn';import { Sessions } from '../../src/agent-session/Sessions';import { InMemorySessionStore } from '../../src/agent-session/store/InMemorySessionStore';+import { TurnNotFoundError } from '../../src/agent-session/store/SessionStoreErrors';import { TurnHandle } from '../../src/agent-session/TurnHandle';import { makeAgentSpec, makeTestResolver, mintTestTurnId } from './testHelpers';@@ -98,6 +100,34 @@ describe('Sessions / SessionHandle / TurnHandle (storage + createTurn)', () => {expect(getSession).not.toHaveBeenCalled();});+ it('freezeTurn cancels a running turn with the given reason', async () => {+ const store = new InMemorySessionStore();+ const sessions = new Sessions({ sessionStore: store });+ const session = await sessions.create({+ tenant_id: tenant,+ session_id: 's1',+ created_by: 'user-1',+ agent: { type: 'inline', spec: makeAgentSpec() },+ });+ const turn = await session.createTurn({+ turn_id: mintTestTurnId(),+ previous_turn_id: 'none',+ signal: new AbortController().signal,+ resolver: makeTestResolver(),+ });+ const frozen = await session.freezeTurn({+ turn_id: turn.id,+ reason: CancellationReason.ClientCancelled,+ });+ expect(frozen.state).toMatchObject({+ status: 'cancelled',+ reason: CancellationReason.ClientCancelled,+ });+ await expect(+ session.freezeTurn({ turn_id: 'missing-turn', reason: CancellationReason.ClientCancelled }),+ ).rejects.toBeInstanceOf(TurnNotFoundError);+ });+it('custom value vs merge-fn', async () => {const store = new InMemorySessionStore<{ tag: string }, { n: number }>();const sessions = new Sessions({ sessionStore: store });diff --git a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.tsindex 5edb06e5b..f67685ae0 100644--- a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts+++ b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts@@ -97,6 +97,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {() =>store.freezeAndGetTurn({...keys,+ reason: CancellationReason.ClientCancelled,turn_done_event: makeTurnDoneEvent(makeCancelledTurnState(CancellationReason.ClientCancelled)),}),() =>@@ -1220,9 +1221,11 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {const record = await store.freezeAndGetTurn({session_id: sessionId,turn_id: 'turn-1',+ reason: CancellationReason.CancelledForNextTurn,turn_done_event: makeTurnDoneEvent(cancelledState),});expect(record.state.status).toBe('cancelled');+ expect(record.state).toMatchObject({ reason: CancellationReason.CancelledForNextTurn });const { data } = await store.listTurnEvents({session_id: sessionId,turn_id: 'turn-1',@@ -1297,6 +1300,23 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {}});+ it('cancels a running turn with the caller-supplied reason', async () => {+ const store = createStore();+ await seedSession(store);+ await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));+ const cancelledState = makeCancelledTurnState(CancellationReason.ClientCancelled);+ const record = await store.freezeAndGetTurn({+ session_id: sessionId,+ turn_id: 'turn-1',+ reason: CancellationReason.ClientCancelled,+ turn_done_event: makeTurnDoneEvent(cancelledState),+ });+ expect(record.state).toMatchObject({+ status: 'cancelled',+ reason: CancellationReason.ClientCancelled,+ });+ });+it('on an already-terminal turn is a plain read without duplicating turn.done', async () => {const store = createStore();await seedSession(store);@@ -1306,6 +1326,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {await store.freezeAndGetTurn({session_id: sessionId,turn_id: 'turn-1',+ reason: CancellationReason.CancelledForNextTurn,turn_done_event: makeTurnDoneEvent(doneState),});const { data } = await store.listTurnEvents({@@ -1327,11 +1348,13 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {store.freezeAndGetTurn({session_id: sessionId,turn_id: 'turn-1',+ reason: CancellationReason.CancelledForNextTurn,turn_done_event: makeTurnDoneEvent(cancelledState),}),store.freezeAndGetTurn({session_id: sessionId,turn_id: 'turn-1',+ reason: CancellationReason.CancelledForNextTurn,turn_done_event: makeTurnDoneEvent(cancelledState),}),]);@@ -1358,6 +1381,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {store.freezeAndGetTurn({session_id: sessionId,turn_id: 'turn-1',+ reason: CancellationReason.CancelledForNextTurn,turn_done_event: makeTurnDoneEvent(cancelledState),}),store.updateTurnState({diff --git a/packages/trueforge/src/apis/sessions.ts b/packages/trueforge/src/apis/sessions.tsindex 5078af7f0..eb92d3657 100644--- a/packages/trueforge/src/apis/sessions.ts+++ b/packages/trueforge/src/apis/sessions.ts@@ -2,21 +2,24 @@* DB-backed sessions API (mounted at /api/v1/sessions).*/import { OpenAPIHono, type RouteHandler } from '@hono/zod-openapi';-import type { ISessionStore, SessionRecord, Sessions } from '@truefoundry/trueforge-core/agent-session';+import type { ISessionStore, SessionHandle, SessionRecord, Sessions } from '@truefoundry/trueforge-core/agent-session';import {CancellationReason,SessionStoreConflictError,SessionStoreInvariantError,SessionStoreNotFoundError,+ TurnNotFoundError,} from '@truefoundry/trueforge-core/agent-session';-import type {- RouteHandler as RequestReplyRouteHandler,- RequestReplyRouter,+import { extractErrorLogFields } from '@truefoundry/trueforge-core/core';+import {+ redisRequest,+ RequestTimeoutError,+ type RouteHandler as RequestReplyRouteHandler,+ type RequestReplyRouter,} from '@truefoundry/trueforge-core/request-reply';-import { NoResponderError, redisRequest, RequestTimeoutError } from '@truefoundry/trueforge-core/request-reply';-import { HTTPException } from 'hono/http-exception';import type { RedisClientType } from 'redis';import { ulid } from 'ulid';+import type { Logger } from 'winston';import { z } from 'zod';import type { ResolveUserContext } from '../auth/identity';import configuration from '../config';@@ -76,6 +79,7 @@ export interface SessionsRouterDeps {redis?: RedisClientType | undefined;requestReplyRouter: RequestReplyRouter;resolveUserContext: ResolveUserContext;+ logger: Logger;}function cancelTurnOnThisExecutor(@@ -112,23 +116,33 @@ export function cancelSessionTurnPeerHandler(activeTurns: ActiveTurnRegistry): R};}-/** A registry to abort in, durable state to read, and a way to reach peers. */+/** A registry to abort in, a session to freeze, durable state to read, and a way to reach peers. */export interface CancelTurnDeps {activeTurns: ActiveTurnRegistry;+ session: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;sessionStore: Pick<ISessionStore, 'getTurn'>;redis?: RedisClientType | undefined;+ logger: Pick<Logger, 'warn'>;}/*** Cancels the turn wherever it runs: locally or on the owning peer over Redis* request-reply. Callers state the motive; default is a plain client cancel.- * Owner failures throw HTTPException (412 unreachable, 424 timed out).+ *+ * A confirmed abort (this process, or peer HTTP 200) lets TurnHandle persist+ * the terminal state. If abort cannot be confirmed, this replica freezes the+ * turn in the store so the session is not stuck `running`.+ *+ * Redis timeout and Redis/transport failures are not a clean cancellation —+ * the owning replica may still be executing — but the turn is still frozen.+ * Later writes from that replica lose to first-terminal-write-wins.*/export async function cancelSessionTurn(deps: CancelTurnDeps,- input: { sessionId: string; turnId: string; reason?: CancellationReason },+ input: { turnId: string; reason?: CancellationReason },): Promise<void> {- const { sessionId, turnId, reason = CancellationReason.ClientCancelled } = input;+ const { turnId, reason = CancellationReason.ClientCancelled } = input;+ const sessionId = deps.session.session_id;const turn = await deps.sessionStore.getTurn({session_id: sessionId,@@ -141,7 +155,7 @@ export async function cancelSessionTurn(const owner = executorFromTurnId(turnId);// Without a Redis client there is no peer to ask, so an id naming another- // replica falls through to the local lookup and finds nothing.+ // replica falls through to the local lookup and freezes if the run is gone.if (owner !== configuration.EXECUTOR_ID && deps.redis) {try {const reply = await redisRequest<CancelPeerBody>({@@ -156,34 +170,45 @@ export async function cancelSessionTurn(pollIntervalMs: configuration.REDIS_REQUEST_REPLY_POLL_INTERVAL_MS,},});- if (reply.status !== 200 && reply.status !== 412) {- throw new HTTPException(500, { message: 'Failed to cancel turn on the owning executor', cause: reply });+ if (reply.status === 200) {+ return;}} catch (error) {- if (error instanceof NoResponderError) {- throw new HTTPException(412, {- message: `Executor owning the running turn is unreachable: ${owner}`,- cause: error,- });- }+ const fields = {+ sessionId,+ turnId,+ owner,+ ...extractErrorLogFields(error),+ };if (error instanceof RequestTimeoutError) {- throw new HTTPException(424, {- message: 'Timed out waiting for the owning executor to cancel the turn',- cause: error,- });+ deps.logger.warn('Timed out waiting for owning executor to cancel; freezing the running turn', fields);+ } else {+ deps.logger.warn('Failed to reach owning executor over Redis; freezing the running turn', fields);}- if (error instanceof HTTPException) {- throw error;- }- throw new HTTPException(500, {- message: `Unexpected error while cancelling the turn on the owning executor: ${owner}`,- cause: error,- });}+ await freezeTurnIgnoringMissing(deps.session, { turnId, reason });return;}- cancelTurnOnThisExecutor(deps.activeTurns, { sessionId, turnId, reason });+ const aborted = cancelTurnOnThisExecutor(deps.activeTurns, { sessionId, turnId, reason });+ if (!aborted) {+ await freezeTurnIgnoringMissing(deps.session, { turnId, reason });+ }+}++/** Freeze a running turn; missing turns are a no-op (already gone). */+async function freezeTurnIgnoringMissing(+ session: Pick<SessionHandle, 'freezeTurn'>,+ input: { turnId: string; reason: CancellationReason },+): Promise<void> {+ try {+ await session.freezeTurn({ turn_id: input.turnId, reason: input.reason });+ } catch (error) {+ if (error instanceof TurnNotFoundError) {+ return;+ }+ throw error;+ }}const FORBIDDEN_SESSION_ACCESS = 'Only the session creator can access this session';@@ -327,19 +352,19 @@ export function createSessionsRouter(deps: SessionsRouterDeps) {const cancelSessionHandler: RouteHandler<typeof cancelSessionRoute> = async c => {const { session_id: sessionId } = c.req.valid('param');- const record = await deps.sessionStore.getSession({ tenant_id: TENANT_ID, session_id: sessionId });- if (!record) {+ const session = await deps.sessions.get({ tenant_id: TENANT_ID, session_id: sessionId });+ if (!session) {return c.json({ error: { message: `Session not found: ${sessionId}` } }, 404);}- if (!checkSessionAccess({ userRef: deps.resolveUserContext(c).userRef, createdBy: record.created_by })) {+ if (!checkSessionAccess({ userRef: deps.resolveUserContext(c).userRef, createdBy: session.record.created_by })) {return c.json({ error: { message: FORBIDDEN_SESSION_ACCESS } }, 403);}- const turnId = record.last_turn_id;+ const turnId = session.record.last_turn_id;if (!turnId) {return c.json({}, 200);}- await cancelSessionTurn(deps, { sessionId, turnId });+ await cancelSessionTurn({ ...deps, session }, { turnId });return c.json({}, 200);};diff --git a/packages/trueforge/src/app.ts b/packages/trueforge/src/app.tsindex 74994d02b..ee32e5fb4 100644--- a/packages/trueforge/src/app.ts+++ b/packages/trueforge/src/app.ts@@ -272,6 +272,7 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {redis: deps.redis,requestReplyRouter: deps.requestReplyRouter,resolveUserContext: resolveUserContext,+ logger: deps.logger,}),),);diff --git a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts b/packages/trueforge/src/db/postgres/session-store/queries/turns.tsindex 6e5827f2d..f89176048 100644--- a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts+++ b/packages/trueforge/src/db/postgres/session-store/queries/turns.ts@@ -1,6 +1,5 @@import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';import {- CancellationReason,type TerminalTurnState,type TurnInputItem,type TurnState,@@ -604,7 +603,7 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGetreturn await db.transaction().execute(async trx => {const cancelledState: TerminalTurnState = {status: 'cancelled',- reason: CancellationReason.CancelledForNextTurn,+ reason: input.reason,completed_at: new Date().toISOString(),};diff --git a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts b/packages/trueforge/src/db/sqlite/session-store/queries/turns.tsindex ca9a346a0..0c5416127 100644--- a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts+++ b/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts@@ -1,6 +1,5 @@import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';import {- CancellationReason,type TerminalTurnState,type TurnInputItem,type TurnState,@@ -657,7 +656,7 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGetreturn await db.transaction().execute(async trx => {const cancelledState: TerminalTurnState = {status: 'cancelled',- reason: CancellationReason.CancelledForNextTurn,+ reason: input.reason,completed_at: nowIso(),};diff --git a/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts b/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.tsindex c931be0ad..832d68ca5 100644--- a/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts+++ b/packages/trueforge/tests/unit/apis/deletedSessionCrud.test.ts@@ -48,6 +48,7 @@ describe('public CRUD after session deletion', () => {redis: createClient(),requestReplyRouter: new RequestReplyRouter(),resolveUserContext: () => LOCAL_USER_CONTEXT,+ logger: createLogger({ silent: true }),}),);app.route(diff --git a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts b/packages/trueforge/tests/unit/apis/sessionHttp.test.tsindex 5b44fd816..dd40b0027 100644--- a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts+++ b/packages/trueforge/tests/unit/apis/sessionHttp.test.ts@@ -2,6 +2,7 @@ import { OpenAPIHono } from '@hono/zod-openapi';import { AgentSpecSchema, Sessions } from '@truefoundry/trueforge-core/agent-session';import { RequestReplyRouter } from '@truefoundry/trueforge-core/request-reply';import { createClient } from 'redis';+import { createLogger } from 'winston';import { createSessionsRouter, TENANT_ID } from '../../../src/apis/sessions';import { LOCAL_USER_CONTEXT } from '../../../src/auth/identity';import { migrateSqliteToLatest } from '../../../src/db/migrateSqlite';@@ -75,6 +76,7 @@ describe('sessions HTTP agent binding', () => {redis: createClient(),requestReplyRouter: new RequestReplyRouter(),resolveUserContext: () => LOCAL_USER_CONTEXT,+ logger: createLogger({ silent: true }),}),);});diff --git a/packages/trueforge/tests/unit/apis/sessions.test.ts b/packages/trueforge/tests/unit/apis/sessions.test.tsindex 72bba1551..7af166b49 100644--- a/packages/trueforge/tests/unit/apis/sessions.test.ts+++ b/packages/trueforge/tests/unit/apis/sessions.test.ts@@ -1,13 +1,30 @@-import type { ISessionStore, TurnRecord, TurnState } from '@truefoundry/trueforge-core/agent-session';-import { CancellationReason } from '@truefoundry/trueforge-core/agent-session';-import { HTTPException } from 'hono/http-exception';-import { createClient, type RedisClientType } from 'redis';+import type { ISessionStore, SessionHandle, TurnRecord, TurnState } from '@truefoundry/trueforge-core/agent-session';+import { CancellationReason, TurnNotFoundError } from '@truefoundry/trueforge-core/agent-session';+import { NoResponderError, redisRequest, RequestTimeoutError } from '@truefoundry/trueforge-core/request-reply';+import type { RedisClientType } from 'redis';import { cancelSessionTurn } from '../../../src/apis/sessions';import configuration from '../../../src/config';import { ActiveTurnRegistry } from '../../../src/runtime/activeTurns';import { mintPeeredTurnId } from '../../../src/runtime/peeringIds';+jest.mock('@truefoundry/trueforge-core/request-reply', () => {+ const actual = jest.requireActual<typeof import('@truefoundry/trueforge-core/request-reply')>(+ '@truefoundry/trueforge-core/request-reply',+ );+ return {+ ...actual,+ redisRequest: jest.fn(),+ };+});++const redisRequestMock = jest.mocked(redisRequest);+const SESSION_ID = 's1';+const REDIS = {} as RedisClientType;++function silentLogger(): { warn: jest.Mock } {+ return { warn: jest.fn() };+}function turnRecord(turnId: string, state: TurnState): TurnRecord {return {@@ -25,15 +42,37 @@ function turnRecord(turnId: string, state: TurnState): TurnRecord {};}-/** Only `getTurn` is exercised, which is all CancelTurnDeps asks for. */function storeReturning(turn: TurnRecord | undefined): Pick<ISessionStore, 'getTurn'> {return { getTurn: () => Promise.resolve(turn) };}-/** Unconnected, so any attempt to reach a peer fails loudly instead of silently passing. */-function unconnectedRedis(): RedisClientType {- const client: RedisClientType = createClient();- return client;+function sessionHandle(): Pick<SessionHandle, 'session_id' | 'freezeTurn'> {+ return {+ session_id: SESSION_ID,+ freezeTurn: jest.fn().mockResolvedValue(undefined),+ };+}++function cancelDeps(input: {+ activeTurns: ActiveTurnRegistry;+ turn: TurnRecord | undefined;+ session?: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;+ redis?: RedisClientType;+ logger?: { warn: jest.Mock };+}): {+ activeTurns: ActiveTurnRegistry;+ session: Pick<SessionHandle, 'session_id' | 'freezeTurn'>;+ sessionStore: Pick<ISessionStore, 'getTurn'>;+ redis?: RedisClientType;+ logger: { warn: jest.Mock };+} {+ return {+ activeTurns: input.activeTurns,+ session: input.session ?? sessionHandle(),+ sessionStore: storeReturning(input.turn),+ ...(input.redis === undefined ? {} : { redis: input.redis }),+ logger: input.logger ?? silentLogger(),+ };}/** Registers a live run without consuming its stream, mirroring a turn mid-execution. */@@ -52,53 +91,154 @@ function trackRun(registry: ActiveTurnRegistry, turnId: string): AbortController}describe('cancelSessionTurn', () => {- it('aborts a turn running in this process', async () => {+ beforeEach(() => {+ redisRequestMock.mockReset();+ });++ it('aborts a turn running in this process without freezing', async () => {const activeTurns = new ActiveTurnRegistry();const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);const abortController = trackRun(activeTurns, turnId);+ const session = sessionHandle();- await cancelSessionTurn(- {- activeTurns,- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),- },- { sessionId: SESSION_ID, turnId },- );+ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {+ turnId,+ });expect(abortController.signal.aborted).toBe(true);expect(abortController.signal.reason).toBe(CancellationReason.ClientCancelled);+ expect(session.freezeTurn).not.toHaveBeenCalled();+ });++ it('freezes when this executor owns the turn id but the run is gone', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);+ const session = sessionHandle();++ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {+ turnId,+ });++ expect(session.freezeTurn).toHaveBeenCalledWith({+ turn_id: turnId,+ reason: CancellationReason.ClientCancelled,+ });+ });++ it('freezes when the run is not in this process and there is no Redis client', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();++ await cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {+ turnId,+ });++ expect(session.freezeTurn).toHaveBeenCalledWith({+ turn_id: turnId,+ reason: CancellationReason.ClientCancelled,+ });+ });++ it('does not freeze when the owning peer aborts', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();+ redisRequestMock.mockResolvedValue({ status: 200, body: {} });++ await cancelSessionTurn(+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS }),+ { turnId },+ );++ expect(redisRequestMock).toHaveBeenCalled();+ expect(session.freezeTurn).not.toHaveBeenCalled();});- it('does not look for a peer without a Redis client', async () => {+ it('freezes when the owning peer is alive but the turn is not running there', async () => {const activeTurns = new ActiveTurnRegistry();const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();+ redisRequestMock.mockResolvedValue({ status: 412, body: { message: 'Turn is not running on this executor' } });++ await cancelSessionTurn(+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS }),+ { turnId },+ );++ expect(session.freezeTurn).toHaveBeenCalled();+ });++ it('freezes when the owning executor is unreachable', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();+ const logger = silentLogger();+ redisRequestMock.mockRejectedValue(new NoResponderError('other1'));- // A peer hop here would dereference the absent client and throw.await expect(cancelSessionTurn(- {- activeTurns,- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),- },- { sessionId: SESSION_ID, turnId },+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),+ { turnId },),).resolves.toBeUndefined();+ expect(session.freezeTurn).toHaveBeenCalled();+ expect(logger.warn).toHaveBeenCalledWith(+ 'Failed to reach owning executor over Redis; freezing the running turn',+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1' }),+ );});- it('routes to the owning peer when the turn id names another replica', async () => {+ it('freezes when waiting for the owning executor times out', async () => {const activeTurns = new ActiveTurnRegistry();const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();+ const logger = silentLogger();+ redisRequestMock.mockRejectedValue(new RequestTimeoutError(60_000));- // The unconnected client makes the hop fail; the point is that it happened.await expect(cancelSessionTurn(- {- activeTurns,- sessionStore: storeReturning(turnRecord(turnId, { status: 'running' })),- redis: unconnectedRedis(),- },- { sessionId: SESSION_ID, turnId },+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),+ { turnId },),- ).rejects.toBeInstanceOf(HTTPException);+ ).resolves.toBeUndefined();+ expect(session.freezeTurn).toHaveBeenCalled();+ expect(logger.warn).toHaveBeenCalledWith(+ 'Timed out waiting for owning executor to cancel; freezing the running turn',+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1' }),+ );+ });++ it('freezes when Redis itself cannot be reached', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId('other1');+ const session = sessionHandle();+ const logger = silentLogger();+ redisRequestMock.mockRejectedValue(new Error('Redis connection closed'));++ await expect(+ cancelSessionTurn(+ cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session, redis: REDIS, logger }),+ { turnId },+ ),+ ).resolves.toBeUndefined();+ expect(session.freezeTurn).toHaveBeenCalled();+ expect(logger.warn).toHaveBeenCalledWith(+ 'Failed to reach owning executor over Redis; freezing the running turn',+ expect.objectContaining({ sessionId: SESSION_ID, turnId, owner: 'other1', error: 'Redis connection closed' }),+ );+ });++ it('treats a missing turn as a successful cancel', async () => {+ const activeTurns = new ActiveTurnRegistry();+ const turnId = mintPeeredTurnId(configuration.EXECUTOR_ID);+ const session = sessionHandle();+ session.freezeTurn = jest.fn().mockRejectedValue(new TurnNotFoundError(turnId));++ await expect(+ cancelSessionTurn(cancelDeps({ activeTurns, turn: turnRecord(turnId, { status: 'running' }), session }), {+ turnId,+ }),+ ).resolves.toBeUndefined();});});