Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(rpc-proxy): pin the wake-hold deadline before host resolution

Alfredo Lopez51ea9a154b69
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 6f24297b1..b28d49f74 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.10.0',
externalRestApiVersion: '0.1.85',
- rpcProxyVersion: '0.1.104',
+ rpcProxyVersion: '0.1.105',
desktopVersion: '0.10.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index f80d46840..a6e8e0b04 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.104",
+ "version": "0.1.105",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.104",
+ "version": "0.1.105",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index de564ce0d..e7839566d 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.104",
+ "version": "0.1.105",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
index dbc4ac9c0..08077eff2 100644
--- a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
+++ b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts
@@ -6,7 +6,11 @@ import request from 'supertest'
import { config } from '../config.js'
import { apiErrorHandler } from '../errorHandler.js'
import { createRpcRouter } from '../routes/rpc.js'
-import { forwardCancelToHost, forwardHostMessageToHost } from '../services/mcpHostRestService.js'
+import {
+ forwardCancelToHost,
+ forwardHostMessageToHost,
+ forwardTaskResultFromHost,
+} from '../services/mcpHostRestService.js'
// PR #849 R2-M3 / R1-M12 / R1-M13 / NEW-rpx-1 / NEW-sec-4.
//
@@ -52,6 +56,7 @@ vi.mock('../services/controlApiRestService.js', () => controlApiMock)
const ALL_SCOPES = [
'host:approval:write',
'host:model:write',
+ 'host:session:read',
'host:task:read',
'host:message:invoke',
'host:wake:write',
@@ -604,3 +609,134 @@ describe('artifacts list keeps a finite first-attempt deadline', () => {
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
})
+
+// R3-L7: the hold deadline is pinned when the request arrives, before the Host
+// connection is resolved, so a slow resolution is charged to the same hold
+// budget instead of extending it. The wake retry never answers, so the request
+// ends when the hold deadline aborts it.
+const HELD_ROUTES = [
+ {
+ label: 'send message',
+ method: 'post',
+ path: '/rpc/hosts/chatllm/messages',
+ body: { content: 'hello' },
+ upstreamPath: '/v1/runtime/messages',
+ },
+ {
+ label: 'approve',
+ method: 'post',
+ path: '/rpc/hosts/chatllm/approvals/approve',
+ body: { toolCallId: 'tc-1' },
+ upstreamPath: '/v1/runtime/approvals/approve',
+ },
+ {
+ label: 'deny',
+ method: 'post',
+ path: '/rpc/hosts/chatllm/approvals/deny',
+ body: { toolCallId: 'tc-1' },
+ upstreamPath: '/v1/runtime/approvals/deny',
+ },
+ {
+ label: 'set model',
+ method: 'post',
+ path: '/rpc/hosts/chatllm/model',
+ body: { chatId: 'c1', model: 'claude-haiku-4-5' },
+ upstreamPath: '/v1/runtime/model',
+ },
+ {
+ label: 'cancel task',
+ method: 'post',
+ path: '/rpc/hosts/chatllm/tasks/task-1/cancel',
+ body: {},
+ upstreamPath: '/v1/runtime/tasks/task-1/cancel',
+ },
+ {
+ label: 'list sessions',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/sessions',
+ upstreamPath: '/v1/runtime/sessions',
+ },
+ {
+ label: 'session transcript',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/messages',
+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/messages',
+ },
+ {
+ label: 'context breakdown',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/context-breakdown',
+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/context-breakdown',
+ },
+ {
+ label: 'list models',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/models',
+ upstreamPath: '/v1/runtime/models',
+ },
+ {
+ label: 'task result',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/tasks/task-1/result',
+ upstreamPath: '/v1/runtime/tasks/task-1/result',
+ },
+ {
+ label: 'list artifacts',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/artifacts',
+ upstreamPath: '/v1/runtime/artifacts',
+ },
+ {
+ label: 'artifact download',
+ method: 'get',
+ path: '/rpc/hosts/chatllm/artifacts/report.bin/download',
+ upstreamPath: '/v1/runtime/artifacts/report.bin/download',
+ },
+] as const
+
+const HOST_RESOLUTION_LATENCY_MS = 500
+const PINNED_HOLD_MS = 1_600
+
+describe.each(HELD_ROUTES)('R3-L7 $label pins the hold deadline before host resolution', route => {
+ it('a slow host resolution is charged to the hold budget', async () => {
+ config.upstreamTimeoutMs = 30_000
+ config.artifactDownloadTimeoutMs = 30_000
+ config.wakeMaxHoldMs = PINNED_HOLD_MS
+ serviceMock.forwardTaskResultFromHost.mockImplementation(
+ forwardTaskResultFromHost as typeof serviceMock.forwardTaskResultFromHost
+ )
+ serviceMock.forwardCancelToHost.mockImplementation(
+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost
+ )
+ await startUpstream((req, _res, index) => {
+ if (index === 0) dropConnection(req) // first attempt: host down
+ // the wake retry is never answered
+ })
+ const { port } = server!.address() as AddressInfo
+ serviceMock.resolveHostConnectionForUser.mockImplementation(async () => {
+ await new Promise(resolve => setTimeout(resolve, HOST_RESOLUTION_LATENCY_MS))
+ return { name: 'chatllm', url: `http://127.0.0.1:${port}`, headers: {} }
+ })
+
+ const startedAt = Date.now()
+ const call =
+ route.method === 'post'
+ ? request(makeApp()).post(route.path).set('authorization', 'Bearer tok').send(route.body)
+ : request(makeApp()).get(route.path).set('authorization', 'Bearer tok')
+ const res = await call.expect(504)
+ const elapsed = Date.now() - startedAt
+
+ expect(res.body).toEqual({ error: 'Gateway Timeout' })
+ // Witness: the slow resolution ran, the first attempt reached the upstream,
+ // a wake was requested and the retry reached the upstream before the
+ // deadline aborted it.
+ expect(serviceMock.resolveHostConnectionForUser).toHaveBeenCalledTimes(1)
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(seen.map(entry => entry.url)).toEqual([route.upstreamPath, route.upstreamPath])
+ await vi.waitFor(() => expect(seen[1]!.closedEarly).toBe(true))
+ // Pinned at arrival: the whole request fits in the hold budget. Pinned after
+ // the resolution it would take HOST_RESOLUTION_LATENCY_MS longer.
+ expect(elapsed).toBeGreaterThanOrEqual(PINNED_HOLD_MS - 100)
+ expect(elapsed).toBeLessThan(PINNED_HOLD_MS + 400)
+ })
+})
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index 29ef05860..d7be6df61 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -653,6 +653,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'hostRef is required' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -696,6 +697,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -721,6 +723,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'hostRef is required' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -760,6 +763,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -803,6 +807,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'Invalid session pagination query' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -854,6 +859,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: forwardSessionList,
@@ -905,6 +911,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'Invalid session messages pagination query' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -957,6 +964,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: forwardTranscript,
@@ -992,6 +1000,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'Invalid hostRef, agent, or chatId' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1033,6 +1042,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: attempt,
@@ -1145,6 +1155,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'hostRef is required' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1185,6 +1196,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: attempt,
@@ -1214,6 +1226,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'hostRef is required' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1256,6 +1269,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
attemptUpstream: attempt,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1285,6 +1299,7 @@ export function createRpcRouter(): Router {
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1317,6 +1332,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: attemptTaskResult,
@@ -1354,6 +1370,7 @@ export function createRpcRouter(): Router {
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1392,6 +1409,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
attemptUpstream: attemptCancel,
respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),
})
@@ -1426,6 +1444,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'Invalid host reference' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1459,6 +1478,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: attempt,
@@ -1500,6 +1520,7 @@ export function createRpcRouter(): Router {
res.status(400).json({ error: 'Invalid filename' })
return
}
+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMs
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
})
@@ -1579,6 +1600,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
// idempotent GET: a re-issued read cannot duplicate a side effect
retryUntilDeadline: true,
attemptUpstream: attemptDownload,