Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(rpc-proxy): pin the wake-hold deadline before host resolution
Alfredo Lopez51ea9a154b69
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex 6f24297b1..b28d49f74 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.10.0',externalRestApiVersion: '0.1.85',- rpcProxyVersion: '0.1.104',+ rpcProxyVersion: '0.1.105',desktopVersion: '0.10.0',minimumDesktopVersion: '0.1.252',}diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex f80d46840..a6e8e0b04 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.104",+ "version": "0.1.105","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.104",+ "version": "0.1.105","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex de564ce0d..e7839566d 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.104",+ "version": "0.1.105","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.tsindex dbc4ac9c0..08077eff2 100644--- a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts+++ b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts@@ -6,7 +6,11 @@ import request from 'supertest'import { config } from '../config.js'import { apiErrorHandler } from '../errorHandler.js'import { createRpcRouter } from '../routes/rpc.js'-import { forwardCancelToHost, forwardHostMessageToHost } from '../services/mcpHostRestService.js'+import {+ forwardCancelToHost,+ forwardHostMessageToHost,+ forwardTaskResultFromHost,+} from '../services/mcpHostRestService.js'// PR #849 R2-M3 / R1-M12 / R1-M13 / NEW-rpx-1 / NEW-sec-4.//@@ -52,6 +56,7 @@ vi.mock('../services/controlApiRestService.js', () => controlApiMock)const ALL_SCOPES = ['host:approval:write','host:model:write',+ 'host:session:read','host:task:read','host:message:invoke','host:wake:write',@@ -604,3 +609,134 @@ describe('artifacts list keeps a finite first-attempt deadline', () => {expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})})++// R3-L7: the hold deadline is pinned when the request arrives, before the Host+// connection is resolved, so a slow resolution is charged to the same hold+// budget instead of extending it. The wake retry never answers, so the request+// ends when the hold deadline aborts it.+const HELD_ROUTES = [+ {+ label: 'send message',+ method: 'post',+ path: '/rpc/hosts/chatllm/messages',+ body: { content: 'hello' },+ upstreamPath: '/v1/runtime/messages',+ },+ {+ label: 'approve',+ method: 'post',+ path: '/rpc/hosts/chatllm/approvals/approve',+ body: { toolCallId: 'tc-1' },+ upstreamPath: '/v1/runtime/approvals/approve',+ },+ {+ label: 'deny',+ method: 'post',+ path: '/rpc/hosts/chatllm/approvals/deny',+ body: { toolCallId: 'tc-1' },+ upstreamPath: '/v1/runtime/approvals/deny',+ },+ {+ label: 'set model',+ method: 'post',+ path: '/rpc/hosts/chatllm/model',+ body: { chatId: 'c1', model: 'claude-haiku-4-5' },+ upstreamPath: '/v1/runtime/model',+ },+ {+ label: 'cancel task',+ method: 'post',+ path: '/rpc/hosts/chatllm/tasks/task-1/cancel',+ body: {},+ upstreamPath: '/v1/runtime/tasks/task-1/cancel',+ },+ {+ label: 'list sessions',+ method: 'get',+ path: '/rpc/hosts/chatllm/sessions',+ upstreamPath: '/v1/runtime/sessions',+ },+ {+ label: 'session transcript',+ method: 'get',+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/messages',+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/messages',+ },+ {+ label: 'context breakdown',+ method: 'get',+ path: '/rpc/hosts/chatllm/sessions/agent-1/chat-1/context-breakdown',+ upstreamPath: '/v1/runtime/sessions/agent-1/chat-1/context-breakdown',+ },+ {+ label: 'list models',+ method: 'get',+ path: '/rpc/hosts/chatllm/models',+ upstreamPath: '/v1/runtime/models',+ },+ {+ label: 'task result',+ method: 'get',+ path: '/rpc/hosts/chatllm/tasks/task-1/result',+ upstreamPath: '/v1/runtime/tasks/task-1/result',+ },+ {+ label: 'list artifacts',+ method: 'get',+ path: '/rpc/hosts/chatllm/artifacts',+ upstreamPath: '/v1/runtime/artifacts',+ },+ {+ label: 'artifact download',+ method: 'get',+ path: '/rpc/hosts/chatllm/artifacts/report.bin/download',+ upstreamPath: '/v1/runtime/artifacts/report.bin/download',+ },+] as const++const HOST_RESOLUTION_LATENCY_MS = 500+const PINNED_HOLD_MS = 1_600++describe.each(HELD_ROUTES)('R3-L7 $label pins the hold deadline before host resolution', route => {+ it('a slow host resolution is charged to the hold budget', async () => {+ config.upstreamTimeoutMs = 30_000+ config.artifactDownloadTimeoutMs = 30_000+ config.wakeMaxHoldMs = PINNED_HOLD_MS+ serviceMock.forwardTaskResultFromHost.mockImplementation(+ forwardTaskResultFromHost as typeof serviceMock.forwardTaskResultFromHost+ )+ serviceMock.forwardCancelToHost.mockImplementation(+ forwardCancelToHost as typeof serviceMock.forwardCancelToHost+ )+ await startUpstream((req, _res, index) => {+ if (index === 0) dropConnection(req) // first attempt: host down+ // the wake retry is never answered+ })+ const { port } = server!.address() as AddressInfo+ serviceMock.resolveHostConnectionForUser.mockImplementation(async () => {+ await new Promise(resolve => setTimeout(resolve, HOST_RESOLUTION_LATENCY_MS))+ return { name: 'chatllm', url: `http://127.0.0.1:${port}`, headers: {} }+ })++ const startedAt = Date.now()+ const call =+ route.method === 'post'+ ? request(makeApp()).post(route.path).set('authorization', 'Bearer tok').send(route.body)+ : request(makeApp()).get(route.path).set('authorization', 'Bearer tok')+ const res = await call.expect(504)+ const elapsed = Date.now() - startedAt++ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ // Witness: the slow resolution ran, the first attempt reached the upstream,+ // a wake was requested and the retry reached the upstream before the+ // deadline aborted it.+ expect(serviceMock.resolveHostConnectionForUser).toHaveBeenCalledTimes(1)+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(seen.map(entry => entry.url)).toEqual([route.upstreamPath, route.upstreamPath])+ await vi.waitFor(() => expect(seen[1]!.closedEarly).toBe(true))+ // Pinned at arrival: the whole request fits in the hold budget. Pinned after+ // the resolution it would take HOST_RESOLUTION_LATENCY_MS longer.+ expect(elapsed).toBeGreaterThanOrEqual(PINNED_HOLD_MS - 100)+ expect(elapsed).toBeLessThan(PINNED_HOLD_MS + 400)+ })+})diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.tsindex 29ef05860..d7be6df61 100644--- a/rpc-proxy/src/routes/rpc.ts+++ b/rpc-proxy/src/routes/rpc.ts@@ -653,6 +653,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'hostRef is required' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -696,6 +697,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,attemptUpstream: attempt,respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),})@@ -721,6 +723,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'hostRef is required' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -760,6 +763,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,attemptUpstream: attempt,respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),})@@ -803,6 +807,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'Invalid session pagination query' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -854,6 +859,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: forwardSessionList,@@ -905,6 +911,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'Invalid session messages pagination query' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -957,6 +964,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: forwardTranscript,@@ -992,6 +1000,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'Invalid hostRef, agent, or chatId' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1033,6 +1042,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: attempt,@@ -1145,6 +1155,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'hostRef is required' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1185,6 +1196,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: attempt,@@ -1214,6 +1226,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'hostRef is required' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1256,6 +1269,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,attemptUpstream: attempt,respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),})@@ -1285,6 +1299,7 @@ export function createRpcRouter(): Router {return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1317,6 +1332,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: attemptTaskResult,@@ -1354,6 +1370,7 @@ export function createRpcRouter(): Router {return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1392,6 +1409,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,attemptUpstream: attemptCancel,respondLegacy: legacyError => respondUpstreamUnavailable(res, legacyError),})@@ -1426,6 +1444,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'Invalid host reference' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1459,6 +1478,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: attempt,@@ -1500,6 +1520,7 @@ export function createRpcRouter(): Router {res.status(400).json({ error: 'Invalid filename' })return}+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})@@ -1579,6 +1600,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,// idempotent GET: a re-issued read cannot duplicate a side effectretryUntilDeadline: true,attemptUpstream: attemptDownload,