Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(rpc-proxy): retry idempotent GETs until the wake-hold deadline

Alfredo Lopeza9a11cd0e71f
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index bb35bc927..6f24297b1 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.10.0',
externalRestApiVersion: '0.1.85',
- rpcProxyVersion: '0.1.103',
+ rpcProxyVersion: '0.1.104',
desktopVersion: '0.10.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index 3fdbed6fc..f80d46840 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.103",
+ "version": "0.1.104",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.103",
+ "version": "0.1.104",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index 35c37b165..de564ce0d 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.103",
+ "version": "0.1.104",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/wake-and-hold-idempotent-gets.real-http.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-idempotent-gets.real-http.test.ts
new file mode 100644
index 000000000..2f1b8c502
--- /dev/null
+++ b/rpc-proxy/src/__tests__/wake-and-hold-idempotent-gets.real-http.test.ts
@@ -0,0 +1,299 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http'
+import type { AddressInfo } from 'node:net'
+import request from 'supertest'
+import { config } from '../config.js'
+import { apiErrorHandler } from '../errorHandler.js'
+import { createRpcRouter } from '../routes/rpc.js'
+import { forwardCancelToHost, forwardTaskResultFromHost } from '../services/mcpHostRestService.js'
+
+// PR #849 R3-M1: a wake-and-hold retry of an idempotent GET keeps re-issuing
+// the read until the hold deadline, like `/messages`; the mutating routes keep
+// their single re-issue. Every test talks to a REAL http server through the
+// real `fetch`; the requests the upstream records are the liveness witness.
+
+const authTokenMock = vi.hoisted(() => ({ verifyRpcToken: vi.fn() }))
+const serviceMock = vi.hoisted(() => ({
+ listAllowedServersForUser: vi.fn(),
+ resolveServerConnectionForUser: vi.fn(),
+ resolveHostConnectionForUser: vi.fn(),
+ validateRpcRequest: vi.fn(),
+ forwardRpcToServer: vi.fn(),
+ forwardHostMessageToHost: vi.fn(),
+ forwardHostActivity: vi.fn(),
+ forwardHostStatus: vi.fn(),
+ forwardHostHealth: vi.fn(),
+ forwardTaskResultFromHost: vi.fn(),
+ forwardCancelToHost: vi.fn(),
+ UpstreamHostError: class UpstreamHostError extends Error {
+ constructor(
+ public readonly status: number,
+ public readonly bodySnippet: string
+ ) {
+ super(`Upstream host returned ${status}: ${bodySnippet}`)
+ this.name = 'UpstreamHostError'
+ }
+ },
+}))
+const controlApiMock = vi.hoisted(() => ({
+ fetchUserAllowedServersFromControlApi: vi.fn(),
+ fetchHostConnectionFromControlApi: vi.fn(),
+ requestHostWakeFromControlApi: vi.fn(),
+}))
+
+vi.mock('../authToken.js', () => authTokenMock)
+vi.mock('../services/mcpProxyService.js', () => serviceMock)
+vi.mock('../services/controlApiRestService.js', () => controlApiMock)
+
+function claims() {
+ return {
+ sub: 'user-uuid-123',
+ typ: 'user' as const,
+ accessScope: 'team' as const,
+ teamId: 'team-1',
+ scopes: [
+ 'host:approval:write',
+ 'host:model:write',
+ 'host:session:read',
+ 'host:task:read',
+ 'host:message:invoke',
+ 'host:wake:write',
+ ],
+ hostRefs: ['chatllm'],
+ jti: 'j1',
+ iat: 1,
+ exp: 9999999999,
+ }
+}
+
+function makeApp() {
+ const app = express()
+ app.use(express.json())
+ app.use(createRpcRouter())
+ app.use(apiErrorHandler)
+ return app
+}
+
+type SeenRequest = { method: string; url: string }
+type Handler = (req: IncomingMessage, res: ServerResponse, index: number) => void
+
+let server: Server | undefined
+let seen: SeenRequest[] = []
+
+async function startUpstream(handler: Handler): Promise<void> {
+ seen = []
+ server = createServer((req, res) => {
+ seen.push({ method: req.method ?? '', url: req.url ?? '' })
+ handler(req, res, seen.length - 1)
+ })
+ await new Promise<void>((resolve, reject) => {
+ server!.once('error', reject)
+ server!.listen(0, '127.0.0.1', resolve)
+ })
+ const { port } = server.address() as AddressInfo
+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({
+ name: 'chatllm',
+ url: `http://127.0.0.1:${port}`,
+ headers: {},
+ })
+}
+
+/** Ends the TCP connection without a response: undici reports it as `fetch failed`. */
+function dropConnection(req: IncomingMessage): void {
+ req.socket.destroy()
+}
+
+function answerJson(res: ServerResponse, status: number, body: unknown): void {
+ res.writeHead(status, { 'content-type': 'application/json' })
+ res.end(JSON.stringify(body))
+}
+
+/** The first `drops` requests lose their connection; later ones answer 200. */
+function dropThenAnswer(drops: number, body: unknown): Handler {
+ return (req, res, index) => {
+ if (index < drops) {
+ dropConnection(req)
+ return
+ }
+ answerJson(res, 200, body)
+ }
+}
+
+const original = {
+ upstreamTimeoutMs: config.upstreamTimeoutMs,
+ wakeMaxHoldMs: config.wakeMaxHoldMs,
+}
+
+beforeEach(() => {
+ vi.clearAllMocks()
+ authTokenMock.verifyRpcToken.mockReturnValue(claims())
+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({
+ kind: 'active',
+ wakeGeneration: null,
+ })
+ serviceMock.forwardHostHealth.mockResolvedValue({ ok: true })
+ serviceMock.forwardTaskResultFromHost.mockImplementation(
+ forwardTaskResultFromHost as typeof serviceMock.forwardTaskResultFromHost
+ )
+ serviceMock.forwardCancelToHost.mockImplementation(
+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost
+ )
+ config.upstreamTimeoutMs = 2_000
+ config.wakeMaxHoldMs = 4_000
+})
+
+afterEach(async () => {
+ config.upstreamTimeoutMs = original.upstreamTimeoutMs
+ config.wakeMaxHoldMs = original.wakeMaxHoldMs
+ if (server) {
+ server.closeAllConnections()
+ await new Promise<void>(resolve => server!.close(() => resolve()))
+ server = undefined
+ }
+})
+
+const IDEMPOTENT_GET_ROUTES = [
+ {
+ label: 'list sessions',
+ path: '/rpc/hosts/chatllm/sessions',
+ upstreamPath: '/v1/runtime/sessions',
+ },
+ {
+ label: 'session transcript',
+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/messages',
+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/messages',
+ },
+ {
+ label: 'context breakdown',
+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/context-breakdown',
+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/context-breakdown',
+ },
+ {
+ label: 'list models',
+ path: '/rpc/hosts/chatllm/models',
+ upstreamPath: '/v1/runtime/models',
+ },
+ {
+ label: 'task result',
+ path: '/rpc/hosts/chatllm/tasks/task-1/result',
+ upstreamPath: '/v1/runtime/tasks/task-1/result',
+ },
+ {
+ label: 'list artifacts',
+ path: '/rpc/hosts/chatllm/artifacts',
+ upstreamPath: '/v1/runtime/artifacts',
+ },
+ {
+ label: 'artifact download',
+ path: '/rpc/hosts/chatllm/artifacts/report.bin/download',
+ upstreamPath: '/v1/runtime/artifacts/report.bin/download',
+ },
+] as const
+
+describe.each(IDEMPOTENT_GET_ROUTES)('R3-M1 $label retries until the hold deadline', route => {
+ it('re-issues the read past a second dropped connection and relays the answer', async () => {
+ const payload = { route: route.label, ok: true }
+ await startUpstream(dropThenAnswer(2, payload))
+
+ const res = await request(makeApp())
+ .get(route.path)
+ .set('authorization', 'Bearer tok')
+ .buffer(true)
+ .parse((response, callback) => {
+ let text = ''
+ response.setEncoding('utf8')
+ response.on('data', (chunk: string) => (text += chunk))
+ response.on('end', () => callback(null, text))
+ })
+
+ expect(res.status).toBe(200)
+ expect(JSON.parse(res.body as string)).toEqual(payload)
+ // Witness: two dropped reads and the accepted third one all reached the
+ // upstream, and exactly one wake was requested for them.
+ expect(seen).toEqual([
+ { method: 'GET', url: route.upstreamPath },
+ { method: 'GET', url: route.upstreamPath },
+ { method: 'GET', url: route.upstreamPath },
+ ])
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ })
+})
+
+// Paired control: the same upstream behavior on a mutating route keeps the
+// single re-issue (no idempotency key), so the caller gets host_waking with
+// exactly two POSTs and the third never happens.
+const MUTATING_ROUTES = [
+ {
+ label: 'approve',
+ path: '/rpc/hosts/chatllm/approvals/approve',
+ body: { toolCallId: 'tc-1' },
+ upstreamPath: '/v1/runtime/approvals/approve',
+ },
+ {
+ label: 'deny',
+ path: '/rpc/hosts/chatllm/approvals/deny',
+ body: { toolCallId: 'tc-1' },
+ upstreamPath: '/v1/runtime/approvals/deny',
+ },
+ {
+ label: 'set model',
+ path: '/rpc/hosts/chatllm/model',
+ body: { chatId: 'c1', model: 'claude-haiku-4-5' },
+ upstreamPath: '/v1/runtime/model',
+ },
+ {
+ label: 'cancel task',
+ path: '/rpc/hosts/chatllm/tasks/task-1/cancel',
+ body: {},
+ upstreamPath: '/v1/runtime/tasks/task-1/cancel',
+ },
+] as const
+
+describe.each(MUTATING_ROUTES)('R3-M1 control: $label keeps a single re-issue', route => {
+ it('answers host_waking after exactly two POSTs', async () => {
+ await startUpstream(dropThenAnswer(2, { applied: true }))
+
+ const res = await request(makeApp())
+ .post(route.path)
+ .set('authorization', 'Bearer tok')
+ .send(route.body)
+ .expect(503)
+
+ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })
+ // Witness: the wake was requested and the re-issue reached the upstream.
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(seen).toEqual([
+ { method: 'POST', url: route.upstreamPath },
+ { method: 'POST', url: route.upstreamPath },
+ ])
+ })
+})
+
+// Lateral finding: forwardTaskResultFromHost threw its UpstreamHostError with an
+// empty body, so mcp-host's `host_draining` fence was invisible and a draining
+// Host answered 502 without a wake.
+describe('task result sees the host_draining fence', () => {
+ it('wakes and relays the re-issued read after a draining 503', async () => {
+ await startUpstream((_req, res, index) => {
+ if (index === 0) {
+ answerJson(res, 503, { code: 'host_draining' })
+ return
+ }
+ answerJson(res, 200, { taskId: 'task-1', status: 'completed' })
+ })
+
+ const res = await request(makeApp())
+ .get('/rpc/hosts/chatllm/tasks/task-1/result')
+ .set('authorization', 'Bearer tok')
+ .expect(200)
+
+ expect(res.body).toEqual({ taskId: 'task-1', status: 'completed' })
+ // Witness: the draining answer was seen, one wake followed, then one retry.
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(seen.map(entry => `${entry.method} ${entry.url}`)).toEqual([
+ 'GET /v1/runtime/tasks/task-1/result',
+ 'GET /v1/runtime/tasks/task-1/result',
+ ])
+ })
+})
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index 2bce2885b..29ef05860 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -854,6 +854,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: forwardSessionList,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -955,6 +957,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: forwardTranscript,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1029,6 +1033,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1179,6 +1185,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1309,6 +1317,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: attemptTaskResult,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1449,6 +1459,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1567,6 +1579,8 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ // idempotent GET: a re-issued read cannot duplicate a side effect
+ retryUntilDeadline: true,
attemptUpstream: attemptDownload,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
diff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.ts
index 6756be814..7f29746bd 100644
--- a/rpc-proxy/src/services/mcpHostRestService.ts
+++ b/rpc-proxy/src/services/mcpHostRestService.ts
@@ -221,8 +221,10 @@ export async function forwardTaskResultFromHost(
}
)
if (response.status === 404) return null
- if (!response.ok) throw new UpstreamHostError(response.status, '')
const rawBody = await response.text()
+ // The body carries mcp-host's `host_draining` fence, which is what makes a
+ // draining 503 wake-eligible (isHostDrainingError reads bodySnippet).
+ if (!response.ok) throw new UpstreamHostError(response.status, rawBody.slice(0, 300))
if (!rawBody.trim()) return null
try {
const parsed = JSON.parse(rawBody) as unknown
diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.ts
index 672b85683..914dde3db 100644
--- a/rpc-proxy/src/services/wakeAndHold.ts
+++ b/rpc-proxy/src/services/wakeAndHold.ts
@@ -655,10 +655,12 @@ export type RespondWithWakeAndHoldOptions = {
deadlineMs?: number
/**
* Keep re-issuing the request (with backoff) until it is accepted or the
- * deadline expires. Only safe when the upstream dedupes a re-presented
- * request: `/messages` carries a per-request `messageId` that mcp-host's
- * admission sink dedupes. Every other route re-issues at most ONCE after the
- * hold: approve/deny/model/cancel have no idempotency key, so each extra
+ * deadline expires. Only safe when a re-issued request cannot duplicate a
+ * side effect: the idempotent GET reads (sessions, transcript, context
+ * breakdown, models, task result, artifacts list and download, which commits
+ * only at `res.send`), and `/messages`, whose per-request `messageId`
+ * mcp-host's admission sink dedupes. Approve/deny/model/cancel have no
+ * idempotency key, so they re-issue at most ONCE after the hold: each extra
* POST risks a duplicate side effect. Defaults to false.
*/
retryUntilDeadline?: boolean