Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(rpc-proxy): abort mutating upstream calls when the client disconnects

Alfredo Lopezab9240721cc5
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index b28d49f74..1784fe811 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.10.0',
externalRestApiVersion: '0.1.85',
- rpcProxyVersion: '0.1.105',
+ rpcProxyVersion: '0.1.106',
desktopVersion: '0.10.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index a6e8e0b04..c379d0443 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.105",
+ "version": "0.1.106",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.105",
+ "version": "0.1.106",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index e7839566d..011d3b2b5 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.105",
+ "version": "0.1.106",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/approval-artifacts.test.ts b/rpc-proxy/src/__tests__/approval-artifacts.test.ts
index 6e410a88b..f29b3f84f 100644
--- a/rpc-proxy/src/__tests__/approval-artifacts.test.ts
+++ b/rpc-proxy/src/__tests__/approval-artifacts.test.ts
@@ -133,11 +133,18 @@ describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {
// attempt would answer 504 for an approval the host may already have applied.
// Real-socket coverage (slow upstream, hung retry) lives in
// route-upstream-deadlines.real-http.test.ts.
- it('sends the first approval attempt without a client-side abort signal', async () => {
+ it('sends the first approval attempt without a client-side deadline', async () => {
config.upstreamTimeoutMs = 20
- const fetchMock = vi
- .fn()
- .mockImplementation(async () => mockFetchResponse(200, JSON.stringify({ ok: true })))
+ // The first attempt carries only the client-disconnect signal (R3-L6): it
+ // must outlive several upstream timeouts without being aborted.
+ let firstSignal: AbortSignal | undefined
+ let abortedWhenAnswered: boolean | undefined
+ const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {
+ firstSignal = init.signal as AbortSignal
+ await new Promise(resolve => setTimeout(resolve, config.upstreamTimeoutMs * 3))
+ abortedWhenAnswered = firstSignal.aborted
+ return mockFetchResponse(200, JSON.stringify({ ok: true }))
+ })
globalThis.fetch = fetchMock
await request(makeApp())
@@ -147,7 +154,8 @@ describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {
.expect(200)
expect(fetchMock).toHaveBeenCalledTimes(1)
- expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()
+ expect(firstSignal).toBeInstanceOf(AbortSignal)
+ expect(abortedWhenAnswered).toBe(false)
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
diff --git a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
index 08077eff2..e5dfc21d4 100644
--- a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
+++ b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
@@ -1,6 +1,12 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
-import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http'
+import {
+ type IncomingMessage,
+ type Server,
+ type ServerResponse,
+ createServer,
+ request as httpRequest,
+} from 'node:http'
import type { AddressInfo } from 'node:net'
import request from 'supertest'
import { config } from '../config.js'
@@ -740,3 +746,105 @@ describe.each(HELD_ROUTES)('R3-L7 $label pins the hold deadline before host reso
expect(elapsed).toBeLessThan(PINNED_HOLD_MS + 400)
})
})
+
+// R3-L6: the first attempt of a mutating call has no client-side timeout (an
+// abort after the upstream applied it would answer 504 for a change that took
+// effect), but a client that is gone must still release the upstream socket.
+const DISCONNECT_ROUTES = [
+ ...MUTATING_ROUTES,
+ {
+ label: 'cancel task',
+ path: '/rpc/hosts/chatllm/tasks/task-1/cancel',
+ body: {},
+ upstreamPath: '/v1/runtime/tasks/task-1/cancel',
+ },
+] as const
+
+const CLIENT_GONE_RELEASE_MS = 500
+
+describe.each(DISCONNECT_ROUTES)(
+ 'R3-L6 $label releases the upstream call when the client disconnects',
+ route => {
+ it('a client that goes away closes the in-flight upstream socket', async () => {
+ serviceMock.forwardCancelToHost.mockImplementation(
+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost
+ )
+ await startUpstream(() => {
+ // never answered: only the client disconnect can end the upstream call
+ })
+ const proxy = makeApp().listen(0, '127.0.0.1')
+ await new Promise<void>((resolve, reject) => {
+ proxy.once('listening', resolve)
+ proxy.once('error', reject)
+ })
+ try {
+ const { port } = proxy.address() as AddressInfo
+ const payload = JSON.stringify(route.body)
+ const clientErrors: string[] = []
+ const clientReq = httpRequest({
+ host: '127.0.0.1',
+ port,
+ method: 'POST',
+ path: route.path,
+ headers: {
+ authorization: 'Bearer tok',
+ 'content-type': 'application/json',
+ 'content-length': Buffer.byteLength(payload),
+ },
+ })
+ // Recorded, not ignored: destroying the request is the event under test.
+ clientReq.on('error', error => clientErrors.push(error.message))
+ clientReq.end(payload)
+
+ // Witness: the POST reached the upstream and is in flight.
+ await vi.waitFor(() => expect(seen).toHaveLength(1), { timeout: 2_000, interval: 10 })
+ expect(seen[0]!.closedEarly).toBe(false)
+
+ clientReq.destroy()
+ const disconnectedAt = Date.now()
+ await vi.waitFor(() => expect(seen[0]!.closedEarly).toBe(true), {
+ timeout: CLIENT_GONE_RELEASE_MS,
+ interval: 10,
+ })
+ expect(Date.now() - disconnectedAt).toBeLessThan(CLIENT_GONE_RELEASE_MS)
+ expect(clientErrors).toEqual(['socket hang up'])
+ expect(seen.map(entry => `${entry.method} ${entry.url}`)).toEqual([
+ `POST ${route.upstreamPath}`,
+ ])
+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ } finally {
+ proxy.closeAllConnections()
+ await new Promise<void>(resolve => proxy.close(() => resolve()))
+ }
+ })
+ }
+)
+
+// R3-L4: task cancel is mutating like approve/deny/model, so its first attempt
+// has no client-side timeout either: a slow upstream answer is relayed.
+describe('R3-L4 task cancel first attempt has no client-side timeout', () => {
+ it('relays an upstream answer that arrives after upstreamTimeoutMs', async () => {
+ config.upstreamTimeoutMs = 200
+ serviceMock.forwardCancelToHost.mockImplementation(
+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost
+ )
+ await startUpstream((_req, res) => {
+ setTimeout(() => {
+ res.writeHead(200, { 'content-type': 'application/json' })
+ res.end(JSON.stringify({ cancelled: true }))
+ }, 400)
+ })
+
+ const res = await request(makeApp())
+ .post('/rpc/hosts/chatllm/tasks/task-1/cancel')
+ .set('authorization', 'Bearer tok')
+ .expect(200)
+
+ expect(res.body).toEqual({ cancelled: true })
+ // Witness: exactly one POST, answered in full by the upstream.
+ expect(seen).toEqual([
+ { method: 'POST', url: '/v1/runtime/tasks/task-1/cancel', closedEarly: false },
+ ])
+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ })
+})
diff --git a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts b/rpc-proxy/src/__tests__/wake-route-coverage.test.ts
index 7643ebdb5..c68ff812e 100644
--- a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts
+++ b/rpc-proxy/src/__tests__/wake-route-coverage.test.ts
@@ -136,12 +136,19 @@ describe('first-attempt upstream deadlines', () => {
path: '/rpc/hosts/chatllm/model',
body: { chatId: 'c1', model: 'claude-haiku-4-5' },
},
- ])('$label sends its first attempt without an abort signal', async row => {
+ ])('$label sends its first attempt without a client-side deadline', async row => {
config.upstreamTimeoutMs = 30
authTokenMock.verifyRpcToken.mockReturnValue(claims([row.scope, 'host:wake:write']))
- const fetchMock = vi
- .fn()
- .mockResolvedValue(mockFetchResponse(200, JSON.stringify({ ok: true })))
+ // The first attempt carries only the client-disconnect signal (R3-L6): it
+ // must outlive several upstream timeouts without being aborted.
+ let firstSignal: AbortSignal | undefined
+ let abortedWhenAnswered: boolean | undefined
+ const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {
+ firstSignal = init.signal as AbortSignal
+ await new Promise(resolve => setTimeout(resolve, config.upstreamTimeoutMs * 3))
+ abortedWhenAnswered = firstSignal.aborted
+ return mockFetchResponse(200, JSON.stringify({ ok: true }))
+ })
globalThis.fetch = fetchMock as unknown as typeof fetch
await request(makeApp())
@@ -151,7 +158,8 @@ describe('first-attempt upstream deadlines', () => {
.expect(200)
expect(fetchMock).toHaveBeenCalledTimes(1)
- expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()
+ expect(firstSignal).toBeInstanceOf(AbortSignal)
+ expect(abortedWhenAnswered).toBe(false)
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index d7be6df61..75710c4c7 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -188,14 +188,22 @@ export function respondControlApiHostAccessRejection(
}
/**
- * Abort signal for a MUTATING upstream call (approve, deny, set-model). The
- * first attempt has no client-side timeout: aborting a request the upstream may
- * already have applied would answer 504 for a change that took effect, and the
- * user's retry would then hit "No pending approval". Only a wake-and-hold retry
- * passes `timeoutMs`, bounded by the hold deadline.
+ * Abort signal for a MUTATING upstream call (approve, deny, set-model, task
+ * cancel). The first attempt has no client-side timeout by design: aborting a
+ * request the upstream may already have applied would answer 504 for a change
+ * that took effect, and the user's retry would then hit "No pending approval".
+ * Only a wake-and-hold retry passes `timeoutMs`, bounded by the hold deadline.
+ * A client that disconnects before the response is written aborts the call on
+ * every attempt, so a gone client never pins the upstream socket.
*/
-function mutatingCallSignal(timeoutMs: number | undefined): AbortSignal | undefined {
- return timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs)
+function mutatingCallSignal(timeoutMs: number | undefined, res: ExpressResponse): AbortSignal {
+ const clientGone = new AbortController()
+ res.once('close', () => {
+ if (!res.writableFinished) clientGone.abort(new Error('client disconnected'))
+ })
+ return timeoutMs === undefined
+ ? clientGone.signal
+ : AbortSignal.any([clientGone.signal, AbortSignal.timeout(timeoutMs)])
}
class ProxyArtifactTooLargeError extends Error {
@@ -680,7 +688,7 @@ export function createRpcRouter(): Router {
method: 'POST',
headers: { 'content-type': 'application/json', ...host.headers },
body: JSON.stringify(upstreamBody),
- signal: mutatingCallSignal(timeoutMs),
+ signal: mutatingCallSignal(timeoutMs, res),
})
const body = await readMutatingResponseBody(response)
const draining = sessionDrainingFence(response, body)
@@ -746,7 +754,7 @@ export function createRpcRouter(): Router {
method: 'POST',
headers: { 'content-type': 'application/json', ...host.headers },
body: JSON.stringify(upstreamBody),
- signal: mutatingCallSignal(timeoutMs),
+ signal: mutatingCallSignal(timeoutMs, res),
})
const body = await readMutatingResponseBody(response)
const draining = sessionDrainingFence(response, body)
@@ -1247,7 +1255,7 @@ export function createRpcRouter(): Router {
method: 'POST',
headers: { 'content-type': 'application/json', ...host.headers },
body: JSON.stringify(body),
- signal: mutatingCallSignal(timeoutMs),
+ signal: mutatingCallSignal(timeoutMs, res),
})
const upstreamBody = await readMutatingResponseBody(response)
const draining = sessionDrainingFence(response, upstreamBody)
@@ -1383,8 +1391,13 @@ export function createRpcRouter(): Router {
`[RPC_PROXY] user=${auth.sub} host=${hostRef} method=cancel-task taskId=${taskId}`
)
- const attemptCancel = async (timeoutMs = config.upstreamTimeoutMs) => {
- const result = await forwardCancelToHost(host, taskId, auth.sub, timeoutMs)
+ const attemptCancel = async (timeoutMs?: number) => {
+ const result = await forwardCancelToHost(
+ host,
+ taskId,
+ auth.sub,
+ mutatingCallSignal(timeoutMs, res)
+ )
if (result.body) {
sendUpstreamBody(
res,
diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
index 59157fa72..80906ad1d 100644
--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
@@ -66,11 +66,6 @@ describe('upstream forwarders abort with the standard classified reason', () =>
timeoutMs: 640,
call: () => forwardTaskResultFromHost(HOST, 'task-1', 640),
},
- {
- label: 'forwardCancelToHost',
- timeoutMs: 530,
- call: () => forwardCancelToHost(HOST, 'task-1', 'user-1', 530),
- },
{
label: 'forwardHostStatus',
timeoutMs: config.upstreamTimeoutMs,
@@ -130,6 +125,52 @@ describe('upstream forwarders abort with the standard classified reason', () =>
)
})
+// R3-L4 / R3-L6: task cancel is a mutating POST with no idempotency key, so it
+// owns no deadline. The route passes one signal that carries the wake-hold
+// deadline (retry only) and the client's disconnect.
+describe('forwardCancelToHost aborts only through the caller signal', () => {
+ const HOST = { name: 'chatllm', url: 'http://chatllm:8080', headers: {} }
+
+ it('stays in flight past the upstream timeout and rejects with the caller abort reason', async () => {
+ vi.useFakeTimers()
+ let signal: AbortSignal | undefined
+ const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(
+ (_input, init) =>
+ new Promise((_resolve, reject) => {
+ signal = init?.signal as AbortSignal
+ signal.addEventListener('abort', () => reject(signal!.reason))
+ })
+ )
+ try {
+ const controller = new AbortController()
+ const pending = forwardCancelToHost(HOST, 'task-1', 'user-1', controller.signal)
+ const rejection = pending.then(
+ () => {
+ throw new Error('expected the forwarder to reject when the caller aborts')
+ },
+ (error: unknown) => error
+ )
+
+ // Liveness witness: the POST is in flight on the caller's own signal, and
+ // four upstream timeouts later nothing has aborted it.
+ await vi.advanceTimersByTimeAsync(config.upstreamTimeoutMs * 4)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(signal).toBe(controller.signal)
+ expect(signal?.aborted).toBe(false)
+
+ const reason = new DOMException('The operation was aborted.', 'AbortError')
+ controller.abort(reason)
+ const error = await rejection
+
+ expect(error).toBe(reason)
+ expect(isUpstreamTimeoutError(error)).toBe(true)
+ } finally {
+ fetchMock.mockRestore()
+ vi.useRealTimers()
+ }
+ })
+})
+
const baseUpstream = {
agent: {
state: 'idle',
diff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.ts
index 7f29746bd..c28b6afff 100644
--- a/rpc-proxy/src/services/mcpHostRestService.ts
+++ b/rpc-proxy/src/services/mcpHostRestService.ts
@@ -245,41 +245,37 @@ export type CancelUpstreamResult = {
contentType: string | null
}
+/**
+ * Task cancel is a mutating POST without an idempotency key, so it owns no
+ * deadline: the caller's `signal` carries the client disconnect and, on a
+ * wake-and-hold retry only, the hold deadline (rpc.ts `mutatingCallSignal`).
+ */
export async function forwardCancelToHost(
host: ResolvedServerConnection,
taskId: string,
- userId?: string,
- timeoutMs = config.upstreamTimeoutMs
+ userId: string | undefined,
+ signal: AbortSignal
): Promise<CancelUpstreamResult> {
const baseUrl = host.url.replace(/\/+$/, '')
- const abortController = new AbortController()
- const timeout = setTimeout(() => abortController.abort(), timeoutMs)
// Mirrors the approve/deny pattern: always forward userId so mcp-host can
// apply the ownership check. Falls back to 'desktop-app' when absent.
const upstreamBody = { userId: userId || 'desktop-app' }
- try {
- const response = await fetch(
- `${baseUrl}/v1/runtime/tasks/${encodeURIComponent(taskId)}/cancel`,
- {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- ...host.headers,
- },
- body: JSON.stringify(upstreamBody),
- signal: abortController.signal,
- }
- )
- // Cancel is a mutating POST without an idempotency key: a body failure
- // after the headers must not re-issue it through the wake path.
- const body = await readMutatingResponseBody(response)
- return {
- status: response.status,
- body,
- contentType: response.headers.get('content-type'),
- }
- } finally {
- clearTimeout(timeout)
+ const response = await fetch(`${baseUrl}/v1/runtime/tasks/${encodeURIComponent(taskId)}/cancel`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ ...host.headers,
+ },
+ body: JSON.stringify(upstreamBody),
+ signal,
+ })
+ // Cancel is a mutating POST without an idempotency key: a body failure
+ // after the headers must not re-issue it through the wake path.
+ const body = await readMutatingResponseBody(response)
+ return {
+ status: response.status,
+ body,
+ contentType: response.headers.get('content-type'),
}
}