Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(rpc-proxy): abort mutating upstream calls when the client disconnects
Alfredo Lopezab9240721cc5
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex b28d49f74..1784fe811 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.10.0',externalRestApiVersion: '0.1.85',- rpcProxyVersion: '0.1.105',+ rpcProxyVersion: '0.1.106',desktopVersion: '0.10.0',minimumDesktopVersion: '0.1.252',}diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex a6e8e0b04..c379d0443 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.105",+ "version": "0.1.106","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.105",+ "version": "0.1.106","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex e7839566d..011d3b2b5 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.105",+ "version": "0.1.106","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/__tests__/approval-artifacts.test.ts b/rpc-proxy/src/__tests__/approval-artifacts.test.tsindex 6e410a88b..f29b3f84f 100644--- a/rpc-proxy/src/__tests__/approval-artifacts.test.ts+++ b/rpc-proxy/src/__tests__/approval-artifacts.test.ts@@ -133,11 +133,18 @@ describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {// attempt would answer 504 for an approval the host may already have applied.// Real-socket coverage (slow upstream, hung retry) lives in// route-upstream-deadlines.real-http.test.ts.- it('sends the first approval attempt without a client-side abort signal', async () => {+ it('sends the first approval attempt without a client-side deadline', async () => {config.upstreamTimeoutMs = 20- const fetchMock = vi- .fn()- .mockImplementation(async () => mockFetchResponse(200, JSON.stringify({ ok: true })))+ // The first attempt carries only the client-disconnect signal (R3-L6): it+ // must outlive several upstream timeouts without being aborted.+ let firstSignal: AbortSignal | undefined+ let abortedWhenAnswered: boolean | undefined+ const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {+ firstSignal = init.signal as AbortSignal+ await new Promise(resolve => setTimeout(resolve, config.upstreamTimeoutMs * 3))+ abortedWhenAnswered = firstSignal.aborted+ return mockFetchResponse(200, JSON.stringify({ ok: true }))+ })globalThis.fetch = fetchMockawait request(makeApp())@@ -147,7 +154,8 @@ describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {.expect(200)expect(fetchMock).toHaveBeenCalledTimes(1)- expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()+ expect(firstSignal).toBeInstanceOf(AbortSignal)+ expect(abortedWhenAnswered).toBe(false)expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})diff --git a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.tsindex 08077eff2..e5dfc21d4 100644--- a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts+++ b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts@@ -1,6 +1,12 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'-import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http'+import {+ type IncomingMessage,+ type Server,+ type ServerResponse,+ createServer,+ request as httpRequest,+} from 'node:http'import type { AddressInfo } from 'node:net'import request from 'supertest'import { config } from '../config.js'@@ -740,3 +746,105 @@ describe.each(HELD_ROUTES)('R3-L7 $label pins the hold deadline before host resoexpect(elapsed).toBeLessThan(PINNED_HOLD_MS + 400)})})++// R3-L6: the first attempt of a mutating call has no client-side timeout (an+// abort after the upstream applied it would answer 504 for a change that took+// effect), but a client that is gone must still release the upstream socket.+const DISCONNECT_ROUTES = [+ ...MUTATING_ROUTES,+ {+ label: 'cancel task',+ path: '/rpc/hosts/chatllm/tasks/task-1/cancel',+ body: {},+ upstreamPath: '/v1/runtime/tasks/task-1/cancel',+ },+] as const++const CLIENT_GONE_RELEASE_MS = 500++describe.each(DISCONNECT_ROUTES)(+ 'R3-L6 $label releases the upstream call when the client disconnects',+ route => {+ it('a client that goes away closes the in-flight upstream socket', async () => {+ serviceMock.forwardCancelToHost.mockImplementation(+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost+ )+ await startUpstream(() => {+ // never answered: only the client disconnect can end the upstream call+ })+ const proxy = makeApp().listen(0, '127.0.0.1')+ await new Promise<void>((resolve, reject) => {+ proxy.once('listening', resolve)+ proxy.once('error', reject)+ })+ try {+ const { port } = proxy.address() as AddressInfo+ const payload = JSON.stringify(route.body)+ const clientErrors: string[] = []+ const clientReq = httpRequest({+ host: '127.0.0.1',+ port,+ method: 'POST',+ path: route.path,+ headers: {+ authorization: 'Bearer tok',+ 'content-type': 'application/json',+ 'content-length': Buffer.byteLength(payload),+ },+ })+ // Recorded, not ignored: destroying the request is the event under test.+ clientReq.on('error', error => clientErrors.push(error.message))+ clientReq.end(payload)++ // Witness: the POST reached the upstream and is in flight.+ await vi.waitFor(() => expect(seen).toHaveLength(1), { timeout: 2_000, interval: 10 })+ expect(seen[0]!.closedEarly).toBe(false)++ clientReq.destroy()+ const disconnectedAt = Date.now()+ await vi.waitFor(() => expect(seen[0]!.closedEarly).toBe(true), {+ timeout: CLIENT_GONE_RELEASE_MS,+ interval: 10,+ })+ expect(Date.now() - disconnectedAt).toBeLessThan(CLIENT_GONE_RELEASE_MS)+ expect(clientErrors).toEqual(['socket hang up'])+ expect(seen.map(entry => `${entry.method} ${entry.url}`)).toEqual([+ `POST ${route.upstreamPath}`,+ ])+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ } finally {+ proxy.closeAllConnections()+ await new Promise<void>(resolve => proxy.close(() => resolve()))+ }+ })+ }+)++// R3-L4: task cancel is mutating like approve/deny/model, so its first attempt+// has no client-side timeout either: a slow upstream answer is relayed.+describe('R3-L4 task cancel first attempt has no client-side timeout', () => {+ it('relays an upstream answer that arrives after upstreamTimeoutMs', async () => {+ config.upstreamTimeoutMs = 200+ serviceMock.forwardCancelToHost.mockImplementation(+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost+ )+ await startUpstream((_req, res) => {+ setTimeout(() => {+ res.writeHead(200, { 'content-type': 'application/json' })+ res.end(JSON.stringify({ cancelled: true }))+ }, 400)+ })++ const res = await request(makeApp())+ .post('/rpc/hosts/chatllm/tasks/task-1/cancel')+ .set('authorization', 'Bearer tok')+ .expect(200)++ expect(res.body).toEqual({ cancelled: true })+ // Witness: exactly one POST, answered in full by the upstream.+ expect(seen).toEqual([+ { method: 'POST', url: '/v1/runtime/tasks/task-1/cancel', closedEarly: false },+ ])+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ })+})diff --git a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts b/rpc-proxy/src/__tests__/wake-route-coverage.test.tsindex 7643ebdb5..c68ff812e 100644--- a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts+++ b/rpc-proxy/src/__tests__/wake-route-coverage.test.ts@@ -136,12 +136,19 @@ describe('first-attempt upstream deadlines', () => {path: '/rpc/hosts/chatllm/model',body: { chatId: 'c1', model: 'claude-haiku-4-5' },},- ])('$label sends its first attempt without an abort signal', async row => {+ ])('$label sends its first attempt without a client-side deadline', async row => {config.upstreamTimeoutMs = 30authTokenMock.verifyRpcToken.mockReturnValue(claims([row.scope, 'host:wake:write']))- const fetchMock = vi- .fn()- .mockResolvedValue(mockFetchResponse(200, JSON.stringify({ ok: true })))+ // The first attempt carries only the client-disconnect signal (R3-L6): it+ // must outlive several upstream timeouts without being aborted.+ let firstSignal: AbortSignal | undefined+ let abortedWhenAnswered: boolean | undefined+ const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => {+ firstSignal = init.signal as AbortSignal+ await new Promise(resolve => setTimeout(resolve, config.upstreamTimeoutMs * 3))+ abortedWhenAnswered = firstSignal.aborted+ return mockFetchResponse(200, JSON.stringify({ ok: true }))+ })globalThis.fetch = fetchMock as unknown as typeof fetchawait request(makeApp())@@ -151,7 +158,8 @@ describe('first-attempt upstream deadlines', () => {.expect(200)expect(fetchMock).toHaveBeenCalledTimes(1)- expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()+ expect(firstSignal).toBeInstanceOf(AbortSignal)+ expect(abortedWhenAnswered).toBe(false)expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.tsindex d7be6df61..75710c4c7 100644--- a/rpc-proxy/src/routes/rpc.ts+++ b/rpc-proxy/src/routes/rpc.ts@@ -188,14 +188,22 @@ export function respondControlApiHostAccessRejection(}/**- * Abort signal for a MUTATING upstream call (approve, deny, set-model). The- * first attempt has no client-side timeout: aborting a request the upstream may- * already have applied would answer 504 for a change that took effect, and the- * user's retry would then hit "No pending approval". Only a wake-and-hold retry- * passes `timeoutMs`, bounded by the hold deadline.+ * Abort signal for a MUTATING upstream call (approve, deny, set-model, task+ * cancel). The first attempt has no client-side timeout by design: aborting a+ * request the upstream may already have applied would answer 504 for a change+ * that took effect, and the user's retry would then hit "No pending approval".+ * Only a wake-and-hold retry passes `timeoutMs`, bounded by the hold deadline.+ * A client that disconnects before the response is written aborts the call on+ * every attempt, so a gone client never pins the upstream socket.*/-function mutatingCallSignal(timeoutMs: number | undefined): AbortSignal | undefined {- return timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs)+function mutatingCallSignal(timeoutMs: number | undefined, res: ExpressResponse): AbortSignal {+ const clientGone = new AbortController()+ res.once('close', () => {+ if (!res.writableFinished) clientGone.abort(new Error('client disconnected'))+ })+ return timeoutMs === undefined+ ? clientGone.signal+ : AbortSignal.any([clientGone.signal, AbortSignal.timeout(timeoutMs)])}class ProxyArtifactTooLargeError extends Error {@@ -680,7 +688,7 @@ export function createRpcRouter(): Router {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(upstreamBody),- signal: mutatingCallSignal(timeoutMs),+ signal: mutatingCallSignal(timeoutMs, res),})const body = await readMutatingResponseBody(response)const draining = sessionDrainingFence(response, body)@@ -746,7 +754,7 @@ export function createRpcRouter(): Router {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(upstreamBody),- signal: mutatingCallSignal(timeoutMs),+ signal: mutatingCallSignal(timeoutMs, res),})const body = await readMutatingResponseBody(response)const draining = sessionDrainingFence(response, body)@@ -1247,7 +1255,7 @@ export function createRpcRouter(): Router {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(body),- signal: mutatingCallSignal(timeoutMs),+ signal: mutatingCallSignal(timeoutMs, res),})const upstreamBody = await readMutatingResponseBody(response)const draining = sessionDrainingFence(response, upstreamBody)@@ -1383,8 +1391,13 @@ export function createRpcRouter(): Router {`[RPC_PROXY] user=${auth.sub} host=${hostRef} method=cancel-task taskId=${taskId}`)- const attemptCancel = async (timeoutMs = config.upstreamTimeoutMs) => {- const result = await forwardCancelToHost(host, taskId, auth.sub, timeoutMs)+ const attemptCancel = async (timeoutMs?: number) => {+ const result = await forwardCancelToHost(+ host,+ taskId,+ auth.sub,+ mutatingCallSignal(timeoutMs, res)+ )if (result.body) {sendUpstreamBody(res,diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.tsindex 59157fa72..80906ad1d 100644--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts@@ -66,11 +66,6 @@ describe('upstream forwarders abort with the standard classified reason', () =>timeoutMs: 640,call: () => forwardTaskResultFromHost(HOST, 'task-1', 640),},- {- label: 'forwardCancelToHost',- timeoutMs: 530,- call: () => forwardCancelToHost(HOST, 'task-1', 'user-1', 530),- },{label: 'forwardHostStatus',timeoutMs: config.upstreamTimeoutMs,@@ -130,6 +125,52 @@ describe('upstream forwarders abort with the standard classified reason', () =>)})+// R3-L4 / R3-L6: task cancel is a mutating POST with no idempotency key, so it+// owns no deadline. The route passes one signal that carries the wake-hold+// deadline (retry only) and the client's disconnect.+describe('forwardCancelToHost aborts only through the caller signal', () => {+ const HOST = { name: 'chatllm', url: 'http://chatllm:8080', headers: {} }++ it('stays in flight past the upstream timeout and rejects with the caller abort reason', async () => {+ vi.useFakeTimers()+ let signal: AbortSignal | undefined+ const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(+ (_input, init) =>+ new Promise((_resolve, reject) => {+ signal = init?.signal as AbortSignal+ signal.addEventListener('abort', () => reject(signal!.reason))+ })+ )+ try {+ const controller = new AbortController()+ const pending = forwardCancelToHost(HOST, 'task-1', 'user-1', controller.signal)+ const rejection = pending.then(+ () => {+ throw new Error('expected the forwarder to reject when the caller aborts')+ },+ (error: unknown) => error+ )++ // Liveness witness: the POST is in flight on the caller's own signal, and+ // four upstream timeouts later nothing has aborted it.+ await vi.advanceTimersByTimeAsync(config.upstreamTimeoutMs * 4)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(signal).toBe(controller.signal)+ expect(signal?.aborted).toBe(false)++ const reason = new DOMException('The operation was aborted.', 'AbortError')+ controller.abort(reason)+ const error = await rejection++ expect(error).toBe(reason)+ expect(isUpstreamTimeoutError(error)).toBe(true)+ } finally {+ fetchMock.mockRestore()+ vi.useRealTimers()+ }+ })+})+const baseUpstream = {agent: {state: 'idle',diff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.tsindex 7f29746bd..c28b6afff 100644--- a/rpc-proxy/src/services/mcpHostRestService.ts+++ b/rpc-proxy/src/services/mcpHostRestService.ts@@ -245,41 +245,37 @@ export type CancelUpstreamResult = {contentType: string | null}+/**+ * Task cancel is a mutating POST without an idempotency key, so it owns no+ * deadline: the caller's `signal` carries the client disconnect and, on a+ * wake-and-hold retry only, the hold deadline (rpc.ts `mutatingCallSignal`).+ */export async function forwardCancelToHost(host: ResolvedServerConnection,taskId: string,- userId?: string,- timeoutMs = config.upstreamTimeoutMs+ userId: string | undefined,+ signal: AbortSignal): Promise<CancelUpstreamResult> {const baseUrl = host.url.replace(/\/+$/, '')- const abortController = new AbortController()- const timeout = setTimeout(() => abortController.abort(), timeoutMs)// Mirrors the approve/deny pattern: always forward userId so mcp-host can// apply the ownership check. Falls back to 'desktop-app' when absent.const upstreamBody = { userId: userId || 'desktop-app' }- try {- const response = await fetch(- `${baseUrl}/v1/runtime/tasks/${encodeURIComponent(taskId)}/cancel`,- {- method: 'POST',- headers: {- 'content-type': 'application/json',- ...host.headers,- },- body: JSON.stringify(upstreamBody),- signal: abortController.signal,- }- )- // Cancel is a mutating POST without an idempotency key: a body failure- // after the headers must not re-issue it through the wake path.- const body = await readMutatingResponseBody(response)- return {- status: response.status,- body,- contentType: response.headers.get('content-type'),- }- } finally {- clearTimeout(timeout)+ const response = await fetch(`${baseUrl}/v1/runtime/tasks/${encodeURIComponent(taskId)}/cancel`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ ...host.headers,+ },+ body: JSON.stringify(upstreamBody),+ signal,+ })+ // Cancel is a mutating POST without an idempotency key: a body failure+ // after the headers must not re-issue it through the wake path.+ const body = await readMutatingResponseBody(response)+ return {+ status: response.status,+ body,+ contentType: response.headers.get('content-type'),}}