Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(rpc-proxy): mark revoked host access with a code and bound wake holds at 48000 ms
Alfredo Lopez81820e6bfff5
diff --git a/deploy/overlays/minikube/configmaps/rpc-proxy-config.yaml b/deploy/overlays/minikube/configmaps/rpc-proxy-config.yamlindex f873193e0..153c36aa1 100644--- a/deploy/overlays/minikube/configmaps/rpc-proxy-config.yaml+++ b/deploy/overlays/minikube/configmaps/rpc-proxy-config.yaml@@ -35,6 +35,10 @@ data:RPC_PROXY_UPSTREAM_TIMEOUT_MS: "60000"RPC_PROXY_MAX_TOKEN_LENGTH: "4096"RPC_PROXY_ARTIFACT_DOWNLOAD_MAX_MB: "250"+ # Body-transfer budget of one artifact download (the header wait is bounded by+ # RPC_PROXY_UPSTREAM_TIMEOUT_MS). 300 s moves the 250 MB cap above at a+ # sustained 0.84 MB/s; the unset default (60 s) would need 4.2 MB/s.+ RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS: "300000"RPC_PROXY_ALLOWED_METHOD_PATTERN: "^[a-zA-Z0-9_./:-]{1,120}$"RPC_PROXY_STREAM_MAX_LIFETIME_MS: "600000"RPC_PROXY_STREAM_INTERVAL_MS: "3000"diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex aea5c7c54..f9d12294a 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.10.0',externalRestApiVersion: '0.1.84',- rpcProxyVersion: '0.1.100',+ rpcProxyVersion: '0.1.101',desktopVersion: '0.10.0',minimumDesktopVersion: '0.1.252',}diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex e5797758c..1a855a2d2 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.100",+ "version": "0.1.101","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.100",+ "version": "0.1.101","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex 92b7b6b07..38d2b00ce 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.100",+ "version": "0.1.101","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/__tests__/approval-artifacts.test.ts b/rpc-proxy/src/__tests__/approval-artifacts.test.tsindex 63b1f6f2e..6e410a88b 100644--- a/rpc-proxy/src/__tests__/approval-artifacts.test.ts+++ b/rpc-proxy/src/__tests__/approval-artifacts.test.ts@@ -2,8 +2,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import request from 'supertest'import { config } from '../config.js'+import { apiErrorHandler } from '../errorHandler.js'import { createRpcRouter } from '../routes/rpc.js'-import { isUpstreamTimeoutError } from '../services/wakeAndHold.js'// ── Hoisted mocks ───────────────────────────────────────────────────const authTokenMock = vi.hoisted(() => ({@@ -77,16 +77,7 @@ function makeApp() {const app = express()app.use(express.json())app.use(createRpcRouter())- // Generic error handler to avoid unhandled rejections in tests- app.use(- (err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {- if (isUpstreamTimeoutError(err)) {- res.status(504).json({ error: 'Gateway Timeout' })- return- }- res.status(500).json({ error: err instanceof Error ? err.message : 'Internal error' })- }- )+ app.use(apiErrorHandler)return app}@@ -138,24 +129,25 @@ afterEach(() => {// Approval Routes// =====================================================================describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {- it('keeps the upstream timeout on the first approval attempt', async () => {+ // R2-M3: approve mutates state on the host. A client-side abort of the first+ // attempt would answer 504 for an approval the host may already have applied.+ // Real-socket coverage (slow upstream, hung retry) lives in+ // route-upstream-deadlines.real-http.test.ts.+ it('sends the first approval attempt without a client-side abort signal', async () => {config.upstreamTimeoutMs = 20- let signal: AbortSignal | undefined- globalThis.fetch = vi.fn().mockImplementation((_url: string, init: RequestInit) => {- signal = init.signal as AbortSignal- return new Promise((_resolve, reject) => {- signal!.addEventListener('abort', () => reject(signal!.reason), { once: true })- })- })+ const fetchMock = vi+ .fn()+ .mockImplementation(async () => mockFetchResponse(200, JSON.stringify({ ok: true })))+ globalThis.fetch = fetchMock- const app = makeApp()- await request(app)+ await request(makeApp()).post('/rpc/hosts/chatllm/approvals/approve').set('authorization', 'Bearer token')- .send({ toolCallId: 'tc-timeout' })- .expect(504)+ .send({ toolCallId: 'tc-no-deadline' })+ .expect(200)- expect(signal?.aborted).toBe(true)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})@@ -252,7 +244,10 @@ describe('POST /rpc/hosts/:hostRef/approvals/approve', () => {})it('returns 403 when host is not accessible', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app)@@ -371,7 +366,10 @@ describe('POST /rpc/hosts/:hostRef/approvals/deny', () => {})it('returns 403 when host is not accessible', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app)@@ -449,7 +447,10 @@ describe('GET /rpc/hosts/:hostRef/artifacts', () => {})it('returns 403 when host is not accessible', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app)@@ -578,7 +579,10 @@ describe('POST /rpc/hosts/:hostRef/tasks/:taskId/cancel', () => {...VALID_CLAIMS,scopes: ['host:message:invoke'],})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app)@@ -892,7 +896,10 @@ describe('GET /rpc/hosts/:hostRef/artifacts/:filename/download', () => {})it('returns 403 when host is not accessible', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app)diff --git a/rpc-proxy/src/__tests__/config.test.ts b/rpc-proxy/src/__tests__/config.test.tsindex 6c28f4d0f..b1dd95d29 100644--- a/rpc-proxy/src/__tests__/config.test.ts+++ b/rpc-proxy/src/__tests__/config.test.ts@@ -1,8 +1,13 @@-import { describe, expect, it } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'+import { readFileSync } from 'node:fs'+import { resolve } from 'node:path'import {+ MAX_REQUEST_HOLD_MS,+ MAX_TIMER_DELAY_MS,parseArtifactDownloadMaxBytes,parsePositiveIntMs,parseSandboxUiAllowedPorts,+ parseWakeMaxHoldMs,} from '../config.js'describe('parseSandboxUiAllowedPorts', () => {@@ -73,3 +78,135 @@ describe('parsePositiveIntMs (wake-and-hold intervals)', () => {)})})++// NEW-rpx-2: setTimeout / AbortSignal.timeout treat a delay above 2^31-1 as 1 ms,+// so an oversized value would turn a generous deadline into an instant abort.+describe('parsePositiveIntMs timer bound', () => {+ it('accepts the largest delay a timer can honour', () => {+ expect(MAX_TIMER_DELAY_MS).toBe(2_147_483_647)+ expect(parsePositiveIntMs('RPC_PROXY_UPSTREAM_TIMEOUT_MS', '2147483647')).toBe(2_147_483_647)+ })++ it('rejects a value one above the timer bound', () => {+ expect(() => parsePositiveIntMs('RPC_PROXY_UPSTREAM_TIMEOUT_MS', '2147483648')).toThrow(+ /RPC_PROXY_UPSTREAM_TIMEOUT_MS.*at most 2147483647/+ )+ })+})++describe('parseWakeMaxHoldMs', () => {+ afterEach(() => {+ vi.restoreAllMocks()+ })++ it('honours a value within the request hold cap without warning', () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ expect(parseWakeMaxHoldMs('30000')).toBe(30_000)+ expect(parseWakeMaxHoldMs(String(MAX_REQUEST_HOLD_MS))).toBe(MAX_REQUEST_HOLD_MS)+ expect(warn).not.toHaveBeenCalled()+ })++ it('clamps a larger value to the cap and says so loudly at config load', () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ expect(parseWakeMaxHoldMs('90000')).toBe(MAX_REQUEST_HOLD_MS)+ expect(warn).toHaveBeenCalledTimes(1)+ expect(String(warn.mock.calls[0]![0])).toMatch(+ /RPC_PROXY_WAKE_MAX_HOLD_MS=90000.*clamped to 48000/+ )+ })++ it('still rejects an invalid value instead of clamping it', () => {+ expect(() => parseWakeMaxHoldMs('0')).toThrow(/RPC_PROXY_WAKE_MAX_HOLD_MS.*positive integer/)+ expect(() => parseWakeMaxHoldMs('2147483648')).toThrow(/at most 2147483647/)+ })+})++describe('config timeouts loaded from the environment', () => {+ const ENV_KEYS = [+ 'RPC_PROXY_UPSTREAM_TIMEOUT_MS',+ 'RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS',+ 'RPC_PROXY_WAKE_MAX_HOLD_MS',+ ] as const+ const saved = Object.fromEntries(ENV_KEYS.map(key => [key, process.env[key]]))++ afterEach(() => {+ for (const key of ENV_KEYS) {+ if (saved[key] === undefined) delete process.env[key]+ else process.env[key] = saved[key]+ }+ vi.resetModules()+ vi.restoreAllMocks()+ })++ async function loadConfig() {+ vi.resetModules()+ return (await import('../config.js')).config+ }++ it('loads RPC_PROXY_UPSTREAM_TIMEOUT_MS as an integer', async () => {+ process.env.RPC_PROXY_UPSTREAM_TIMEOUT_MS = '45000'+ expect((await loadConfig()).upstreamTimeoutMs).toBe(45_000)+ })++ it.each(['abc', '0', '-1', '1.5', '2147483648'])(+ 'fails loud on RPC_PROXY_UPSTREAM_TIMEOUT_MS=%s',+ async value => {+ process.env.RPC_PROXY_UPSTREAM_TIMEOUT_MS = value+ await expect(loadConfig()).rejects.toThrow(/RPC_PROXY_UPSTREAM_TIMEOUT_MS/)+ }+ )++ it('loads RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS and rejects an oversized one', async () => {+ process.env.RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS = '300000'+ expect((await loadConfig()).artifactDownloadTimeoutMs).toBe(300_000)++ process.env.RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS = '2147483648'+ await expect(loadConfig()).rejects.toThrow(/RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS/)+ })++ it('clamps RPC_PROXY_WAKE_MAX_HOLD_MS above the cap in the loaded config', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ process.env.RPC_PROXY_WAKE_MAX_HOLD_MS = '90000'+ expect((await loadConfig()).wakeMaxHoldMs).toBe(MAX_REQUEST_HOLD_MS)+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('clamped to 48000'))+ })++ it('the built-in default is the hold cap itself, so it needs no clamp and no boot-time warning', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ delete process.env.RPC_PROXY_WAKE_MAX_HOLD_MS+ expect((await loadConfig()).wakeMaxHoldMs).toBe(48_000)+ expect(MAX_REQUEST_HOLD_MS).toBe(48_000)+ // The literal control-api's host-wake rate-limit guard reads must be the+ // effective default (not a larger value that is clamped at load).+ const source = readFileSync(new URL('../config.ts', import.meta.url), 'utf-8')+ expect(source).toMatch(/RPC_PROXY_WAKE_MAX_HOLD_MS\s*\|\|\s*'48000'/)+ expect(warn).not.toHaveBeenCalled()+ })+})++describe('minikube rpc-proxy ConfigMap', () => {+ const overlay = readFileSync(+ resolve(__dirname, '../../../deploy/overlays/minikube/configmaps/rpc-proxy-config.yaml'),+ 'utf8'+ )++ function value(key: string): number {+ const match = overlay.match(new RegExp(`^\\s+${key}:\\s+"(\\d+)"\\s*$`, 'm'))+ if (!match) throw new Error(`${key} is missing from the minikube rpc-proxy ConfigMap`)+ return Number(match[1])+ }++ it('sets an artifact download timeout sized for the 250 MB cap', () => {+ const timeoutMs = value('RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS')+ const maxMb = value('RPC_PROXY_ARTIFACT_DOWNLOAD_MAX_MB')+ expect(maxMb).toBe(250)+ // The 60 s default needs 4.2 MB/s to move 250 MB; the overlay must not.+ const requiredMbPerSecond = maxMb / (timeoutMs / 1000)+ expect(requiredMbPerSecond).toBeLessThanOrEqual(1)+ expect(timeoutMs).toBeGreaterThan(60_000)+ // ...and it must load through the same parser as production.+ expect(parsePositiveIntMs('RPC_PROXY_ARTIFACT_DOWNLOAD_TIMEOUT_MS', String(timeoutMs))).toBe(+ timeoutMs+ )+ })+})diff --git a/rpc-proxy/src/__tests__/errorHandler.test.ts b/rpc-proxy/src/__tests__/errorHandler.test.tsnew file mode 100644index 000000000..ae923027b--- /dev/null+++ b/rpc-proxy/src/__tests__/errorHandler.test.ts@@ -0,0 +1,54 @@+import { describe, expect, it } from 'vitest'+import express from 'express'+import request from 'supertest'+import { createApp } from '../app.js'+import { apiErrorHandler } from '../errorHandler.js'++function appThrowing(error: unknown) {+ const app = express()+ app.get('/boom', (_req, _res, next) => next(error))+ app.use(apiErrorHandler)+ return app+}++function named(name: string): Error {+ const error = new Error(name)+ error.name = name+ return error+}++describe('apiErrorHandler', () => {+ it.each(['AbortError', 'TimeoutError'])('maps %s to 504 Gateway Timeout', async name => {+ const res = await request(appThrowing(named(name))).get('/boom')+ expect(res.status).toBe(504)+ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ })++ it('maps a DOMException abort (what fetch rejects with) to 504', async () => {+ const res = await request(appThrowing(new DOMException('aborted', 'AbortError'))).get('/boom')+ expect(res.status).toBe(504)+ })++ it('maps body-parser entity.too.large to 413', async () => {+ const res = await request(+ appThrowing(Object.assign(new Error('big'), { type: 'entity.too.large' }))+ ).get('/boom')+ expect(res.status).toBe(413)+ expect(res.body).toEqual({ error: 'Payload Too Large' })+ })++ it('maps anything else to 500 with the message', async () => {+ const res = await request(appThrowing(new Error('kaput'))).get('/boom')+ expect(res.status).toBe(500)+ expect(res.body).toEqual({ error: 'Internal Server Error', message: 'kaput' })+ })++ it('is the terminal handler of the real app (tests and production share it)', () => {+ // Express 4 keeps its layer stack on `_router`; the public `router` getter+ // throws a deprecation error.+ const stack = (createApp() as unknown as { _router?: { stack: Array<{ handle: unknown }> } })+ ._router?.stack+ if (!stack) throw new Error('express app exposes no _router stack; update this guard')+ expect(stack.at(-1)!.handle).toBe(apiErrorHandler)+ })+})diff --git a/rpc-proxy/src/__tests__/host-access-denial-code.route.test.ts b/rpc-proxy/src/__tests__/host-access-denial-code.route.test.tsnew file mode 100644index 000000000..1c38aca6c--- /dev/null+++ b/rpc-proxy/src/__tests__/host-access-denial-code.route.test.ts@@ -0,0 +1,308 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import { apiErrorHandler } from '../errorHandler.js'+import { createRpcRouter } from '../routes/rpc.js'+import { createRpcHostActivityStreamRouter } from '../routes/rpcHostActivityStream.js'+import { createRpcHostProgressStreamRouter } from '../routes/rpcHostProgressStream.js'+import { createRpcHostStatusStreamRouter } from '../routes/rpcHostStatusStream.js'++// PR #849 R1-M6 producer side (also R1-L7 on the rpc-proxy side).+//+// rpc-proxy's OWN Host-authorization 403 carries `code`+// (`host_access_revoked` | `host_access_denied`) at every site. Only the JWT+// verifier is mocked: the real routers, the real mcpProxyService and the real+// controlApiRestService run, and the only stubbed boundary is `fetch`. That+// makes the control-api reason -> code mapping observable end to end and lets a+// passthrough mcp-host 403 prove it is NOT rewritten.++const authTokenMock = vi.hoisted(() => ({ verifyRpcToken: vi.fn() }))+vi.mock('../authToken.js', () => authTokenMock)++const DENIED_ERROR = 'Forbidden: user cannot access this host'++// control-api reports why it denied Host access in this response header; its 403+// body is always exactly {"error":"Forbidden"}. Pinned literally so the test+// fails if rpc-proxy and control-api ever disagree on the name.+const REASON_HEADER = 'x-host-access-denial-reason'+const CONTROL_API_403_BODY = { error: 'Forbidden' }++const CLAIMS = {+ sub: 'user-uuid-123',+ typ: 'user' as const,+ accessScope: 'team' as const,+ teamId: 'team-1',+ scopes: [+ 'host:message:invoke',+ 'host:wake:write',+ 'host:approval:write',+ 'host:session:read',+ 'host:session:write',+ 'host:model:write',+ 'host:task:read',+ 'host:activity:read',+ 'host:status:read',+ 'host:health:read',+ ],+ hostRefs: ['chatllm'],+ jti: 'j1',+ iat: 1,+ exp: 9999999999,+}++function makeApp() {+ const app = express()+ app.use(express.json())+ app.use(createRpcHostStatusStreamRouter())+ app.use(createRpcHostActivityStreamRouter())+ app.use(createRpcHostProgressStreamRouter())+ app.use(createRpcRouter())+ app.use(apiErrorHandler)+ return app+}++type ControlApiAnswer = { status: number; body?: unknown; headers?: Record<string, string> }+const originalFetch = globalThis.fetch+let controlApiAnswer: ControlApiAnswer+let controlApiCalls: string[]+let hostCalls: string[]+let hostAnswer: { status: number; body: string }++beforeEach(() => {+ vi.clearAllMocks()+ authTokenMock.verifyRpcToken.mockReturnValue({ ...CLAIMS })+ controlApiCalls = []+ hostCalls = []+ hostAnswer = { status: 200, body: '{}' }+ controlApiAnswer = {+ status: 403,+ body: CONTROL_API_403_BODY,+ headers: { [REASON_HEADER]: 'directory_grant_missing' },+ }+ globalThis.fetch = vi.fn(async (input: string | URL | Request) => {+ const url = String(input)+ if (url.includes('/rpc/access/users/')) {+ controlApiCalls.push(url)+ return new Response(+ controlApiAnswer.body === undefined ? null : JSON.stringify(controlApiAnswer.body),+ {+ status: controlApiAnswer.status,+ headers: { 'content-type': 'application/json', ...controlApiAnswer.headers },+ }+ )+ }+ hostCalls.push(url)+ return new Response(hostAnswer.body, {+ status: hostAnswer.status,+ headers: { 'content-type': 'application/json' },+ })+ }) as unknown as typeof fetch+})++afterEach(() => {+ globalThis.fetch = originalFetch+})++type RouteCase = {+ label: string+ method: 'get' | 'post' | 'patch'+ path: string+ body?: Record<string, unknown>+}++// Every rpc-proxy route that resolves a Host and answers its own authorization+// 403: the 15 in rpc.ts plus the three streams.+const DENYING_ROUTES: RouteCase[] = [+ {+ label: 'messages',+ method: 'post',+ path: '/rpc/hosts/chatllm/messages',+ body: { content: 'hi' },+ },+ { label: 'wake', method: 'post', path: '/rpc/hosts/chatllm/wake' },+ {+ label: 'approve',+ method: 'post',+ path: '/rpc/hosts/chatllm/approvals/approve',+ body: { toolCallId: 'tc-1' },+ },+ {+ label: 'deny',+ method: 'post',+ path: '/rpc/hosts/chatllm/approvals/deny',+ body: { toolCallId: 'tc-1' },+ },+ { label: 'sessions', method: 'get', path: '/rpc/hosts/chatllm/sessions' },+ {+ label: 'session messages',+ method: 'get',+ path: '/rpc/hosts/chatllm/sessions/agent-a/chat-1/messages',+ },+ {+ label: 'context breakdown',+ method: 'get',+ path: '/rpc/hosts/chatllm/sessions/agent-a/chat-1/context-breakdown',+ },+ {+ label: 'session rename',+ method: 'patch',+ path: '/rpc/hosts/chatllm/sessions/agent-a/chat-1/name',+ body: { name: 'renamed' },+ },+ { label: 'models', method: 'get', path: '/rpc/hosts/chatllm/models' },+ {+ label: 'set model',+ method: 'post',+ path: '/rpc/hosts/chatllm/model',+ body: { chatId: 'c1', model: 'claude-haiku-4-5' },+ },+ { label: 'artifacts list', method: 'get', path: '/rpc/hosts/chatllm/artifacts' },+ {+ label: 'artifact download',+ method: 'get',+ path: '/rpc/hosts/chatllm/artifacts/report.pdf/download',+ },+ { label: 'activity', method: 'get', path: '/rpc/hosts/chatllm/activity' },+ { label: 'status', method: 'get', path: '/rpc/hosts/chatllm/status' },+ { label: 'health', method: 'get', path: '/rpc/hosts/chatllm/health' },+ { label: 'status stream', method: 'get', path: '/rpc/hosts/chatllm/status/stream' },+ { label: 'activity stream', method: 'get', path: '/rpc/hosts/chatllm/activity/stream' },+ {+ label: 'progress stream',+ method: 'get',+ path: '/rpc/hosts/chatllm/tasks/task-1/progress/stream',+ },+]++async function send(route: RouteCase) {+ const pending = request(makeApp())[route.method](route.path).set('authorization', 'Bearer tok')+ return route.body ? pending.send(route.body) : pending+}++describe('host authorization 403 carries a machine-readable code at every site', () => {+ it.each(DENYING_ROUTES)(+ '$label: membership/grant revocation -> host_access_revoked',+ async route => {+ const res = await send(route)++ // Witness: the request reached control-api and no host was contacted.+ expect(controlApiCalls).toHaveLength(1)+ expect(hostCalls).toEqual([])+ expect(res.status).toBe(403)+ expect(res.body).toEqual({ error: DENIED_ERROR, code: 'host_access_revoked' })+ }+ )++ it.each(DENYING_ROUTES)('$label: a missing host (404) -> host_access_denied', async route => {+ controlApiAnswer = { status: 404, body: { error: 'not found' } }++ const res = await send(route)++ expect(controlApiCalls).toHaveLength(1)+ expect(hostCalls).toEqual([])+ expect(res.status).toBe(403)+ expect(res.body).toEqual({ error: DENIED_ERROR, code: 'host_access_denied' })+ })+})++describe('control-api 403 reason -> code mapping', () => {+ const status = DENYING_ROUTES.find(route => route.label === 'status')!++ it.each([+ ['team_membership_missing', 'host_access_revoked'],+ ['directory_grant_missing', 'host_access_revoked'],+ ['subject_mismatch', 'host_access_revoked'],+ ['host_disabled', 'host_access_denied'],+ ['host_missing', 'host_access_denied'],+ ['host_claim_missing', 'host_access_denied'],+ ['a_reason_added_later', 'host_access_denied'],+ ])('reason %s -> %s', async (reason, code) => {+ controlApiAnswer = {+ status: 403,+ body: CONTROL_API_403_BODY,+ headers: { [REASON_HEADER]: reason },+ }++ const res = await send(status)++ expect(controlApiCalls).toHaveLength(1)+ expect(res.status).toBe(403)+ expect(res.body).toEqual({ error: DENIED_ERROR, code })+ })++ it.each([+ ['the header is absent', {}],+ ['the header is empty', { [REASON_HEADER]: '' }],+ ])(+ '%s -> host_access_denied (an unreadable reason never proves revocation)',+ async (_l, headers) => {+ controlApiAnswer = { status: 403, body: CONTROL_API_403_BODY, headers }++ const res = await send(status)++ expect(controlApiCalls).toHaveLength(1)+ expect(res.body).toEqual({ error: DENIED_ERROR, code: 'host_access_denied' })+ }+ )++ it.each(['directory_grant_missing', 'team_membership_missing', 'subject_mismatch'])(+ 'a legacy body-only reason %s (no header) is ignored -> host_access_denied',+ async reason => {+ // Proves the 403 body is never read: the same reason that maps to+ // host_access_revoked when sent as the header must not count from the body.+ controlApiAnswer = { status: 403, body: { error: 'Forbidden', reason }, headers: {} }++ const res = await send(status)++ expect(controlApiCalls).toHaveLength(1)+ expect(res.status).toBe(403)+ expect(res.body).toEqual({ error: DENIED_ERROR, code: 'host_access_denied' })+ }+ )+})++describe('passthrough and non-authorization responses are left untouched', () => {+ beforeEach(() => {+ controlApiAnswer = {+ status: 200,+ body: {+ userId: CLAIMS.sub,+ hostRef: 'chatllm',+ url: 'http://chatllm.mcp-host.svc.cluster.local:8080',+ },+ }+ })++ it('an mcp-host 403 body is relayed verbatim, without a `code`', async () => {+ hostAnswer = { status: 403, body: JSON.stringify({ error: 'Forbidden by mcp-host policy' }) }++ const res = await send(DENYING_ROUTES.find(route => route.label === 'approve')!)++ // Witness: the host was contacted, so this 403 is the host's, not ours.+ expect(controlApiCalls).toHaveLength(1)+ expect(hostCalls).toHaveLength(1)+ expect(res.status).toBe(403)+ expect(JSON.parse(res.text)).toEqual({ error: 'Forbidden by mcp-host policy' })+ expect(res.text).not.toContain('"code"')+ })++ it('an mcp-host 403 on the artifact download is relayed verbatim, without a `code`', async () => {+ hostAnswer = { status: 403, body: JSON.stringify({ error: 'artifact not readable' }) }++ const res = await send(DENYING_ROUTES.find(route => route.label === 'artifact download')!)++ expect(hostCalls).toHaveLength(1)+ expect(res.status).toBe(403)+ expect(JSON.parse(res.text)).toEqual({ error: 'artifact not readable' })+ })++ it('control-api 401 is not a Host denial and gets no `code`', async () => {+ controlApiAnswer = { status: 401, body: { error: 'token rejected' } }+ authTokenMock.verifyRpcToken.mockReturnValue({ ...CLAIMS })++ const res = await send(DENYING_ROUTES.find(route => route.label === 'status')!)++ expect(controlApiCalls).toHaveLength(1)+ expect(res.body).not.toHaveProperty('code')+ })+})diff --git a/rpc-proxy/src/__tests__/host-wake-route.test.ts b/rpc-proxy/src/__tests__/host-wake-route.test.tsindex 8527cf66d..b12a9ea26 100644--- a/rpc-proxy/src/__tests__/host-wake-route.test.ts+++ b/rpc-proxy/src/__tests__/host-wake-route.test.ts@@ -109,12 +109,19 @@ beforeEach(() => {describe('POST /rpc/hosts/:hostRef/wake auth', () => {it('rejects a token that cannot access :hostRef exactly like the messages route (403) and never calls the wake plane', async () => {// Same enforcement point as the messages route: control-api refuses the- // host resolution for this user/token → resolveHostConnectionForUser null.- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ // host resolution for this user/token → resolveHostConnectionForUser+ // returns the typed denial.+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const response = await postWake(makeApp()).expect(403)- expect(response.body).toEqual({ error: 'Forbidden: user cannot access this host' })+ expect(response.body).toEqual({+ error: 'Forbidden: user cannot access this host',+ code: 'host_access_denied',+ })expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})diff --git a/rpc-proxy/src/__tests__/model-selection.test.ts b/rpc-proxy/src/__tests__/model-selection.test.tsindex 52886365f..d20672847 100644--- a/rpc-proxy/src/__tests__/model-selection.test.ts+++ b/rpc-proxy/src/__tests__/model-selection.test.ts@@ -135,7 +135,10 @@ describe('GET /rpc/hosts/:hostRef/models — passthrough to mcp-host', () => {})it('returns 403 if the user cannot access the host', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })await request(makeApp()).get('/rpc/hosts/other-host/models').set('authorization', 'Bearer user-token')@@ -241,7 +244,10 @@ describe('POST /rpc/hosts/:hostRef/model — set per-session model', () => {})it('returns 403 if the user cannot access the host', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })await request(makeApp()).post('/rpc/hosts/other-host/model').set('authorization', 'Bearer user-token')diff --git a/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.tsnew file mode 100644index 000000000..aeca45956--- /dev/null+++ b/rpc-proxy/src/__tests__/route-upstream-deadlines.real-http.test.ts@@ -0,0 +1,472 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http'+import type { AddressInfo } from 'node:net'+import request from 'supertest'+import { config } from '../config.js'+import { apiErrorHandler } from '../errorHandler.js'+import { createRpcRouter } from '../routes/rpc.js'+import { forwardHostMessageToHost } from '../services/mcpHostRestService.js'++// PR #849 R2-M3 / R1-M12 / R1-M13 / NEW-rpx-1 / NEW-sec-4.+//+// Every test here talks to a REAL http server through the real `fetch`, so the+// timers under test are the production timers, not a mocked fetch that reacts+// to an AbortSignal. Each upstream handler records the requests it saw: that is+// the liveness witness proving the request reached the upstream before the+// asserted outcome, so a 504 cannot be explained by "the request never left".++const authTokenMock = vi.hoisted(() => ({ verifyRpcToken: vi.fn() }))+const serviceMock = vi.hoisted(() => ({+ listAllowedServersForUser: vi.fn(),+ resolveServerConnectionForUser: vi.fn(),+ resolveHostConnectionForUser: vi.fn(),+ validateRpcRequest: vi.fn(),+ forwardRpcToServer: vi.fn(),+ forwardHostMessageToHost: vi.fn(),+ forwardHostActivity: vi.fn(),+ forwardHostStatus: vi.fn(),+ forwardHostHealth: vi.fn(),+ forwardTaskResultFromHost: vi.fn(),+ forwardCancelToHost: vi.fn(),+ UpstreamHostError: class UpstreamHostError extends Error {+ constructor(+ public readonly status: number,+ public readonly bodySnippet: string+ ) {+ super(`Upstream host returned ${status}: ${bodySnippet}`)+ this.name = 'UpstreamHostError'+ }+ },+}))+const controlApiMock = vi.hoisted(() => ({+ fetchUserAllowedServersFromControlApi: vi.fn(),+ fetchHostConnectionFromControlApi: vi.fn(),+ requestHostWakeFromControlApi: vi.fn(),+}))++vi.mock('../authToken.js', () => authTokenMock)+vi.mock('../services/mcpProxyService.js', () => serviceMock)+vi.mock('../services/controlApiRestService.js', () => controlApiMock)++const ALL_SCOPES = [+ 'host:approval:write',+ 'host:model:write',+ 'host:task:read',+ 'host:message:invoke',+ 'host:wake:write',+]++function claims() {+ return {+ sub: 'user-uuid-123',+ typ: 'user' as const,+ accessScope: 'team' as const,+ teamId: 'team-1',+ scopes: ALL_SCOPES,+ hostRefs: ['chatllm'],+ jti: 'j1',+ iat: 1,+ exp: 9999999999,+ }+}++function makeApp() {+ const app = express()+ app.use(express.json())+ app.use(createRpcRouter())+ app.use(apiErrorHandler)+ return app+}++type SeenRequest = { method: string; url: string; closedEarly: boolean }+type Handler = (req: IncomingMessage, res: ServerResponse, index: number) => void++let server: Server | undefined+let seen: SeenRequest[] = []++async function startUpstream(handler: Handler): Promise<void> {+ seen = []+ server = createServer((req, res) => {+ const entry: SeenRequest = { method: req.method ?? '', url: req.url ?? '', closedEarly: false }+ seen.push(entry)+ res.once('close', () => {+ if (!res.writableFinished) entry.closedEarly = true+ })+ handler(req, res, seen.length - 1)+ })+ await new Promise<void>((resolve, reject) => {+ server!.once('error', reject)+ server!.listen(0, '127.0.0.1', resolve)+ })+ const { port } = server.address() as AddressInfo+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ name: 'chatllm',+ url: `http://127.0.0.1:${port}`,+ headers: {},+ })+}++/** Ends the TCP connection without a response: undici reports it as `fetch failed`. */+function dropConnection(req: IncomingMessage): void {+ req.socket.destroy()+}++const original = {+ upstreamTimeoutMs: config.upstreamTimeoutMs,+ artifactDownloadTimeoutMs: config.artifactDownloadTimeoutMs,+ wakeMaxHoldMs: config.wakeMaxHoldMs,+}++beforeEach(() => {+ vi.clearAllMocks()+ authTokenMock.verifyRpcToken.mockReturnValue(claims())+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({+ kind: 'active',+ wakeGeneration: null,+ })+ serviceMock.forwardHostHealth.mockResolvedValue({ ok: true })+ serviceMock.forwardHostMessageToHost.mockImplementation(+ forwardHostMessageToHost as typeof serviceMock.forwardHostMessageToHost+ )+})++afterEach(async () => {+ config.upstreamTimeoutMs = original.upstreamTimeoutMs+ config.artifactDownloadTimeoutMs = original.artifactDownloadTimeoutMs+ config.wakeMaxHoldMs = original.wakeMaxHoldMs+ if (server) {+ server.closeAllConnections()+ await new Promise<void>(resolve => server!.close(() => resolve()))+ server = undefined+ }+})++const MUTATING_ROUTES = [+ {+ label: 'approve',+ path: '/rpc/hosts/chatllm/approvals/approve',+ body: { toolCallId: 'tc-1' },+ upstreamPath: '/v1/runtime/approvals/approve',+ },+ {+ label: 'deny',+ path: '/rpc/hosts/chatllm/approvals/deny',+ body: { toolCallId: 'tc-1' },+ upstreamPath: '/v1/runtime/approvals/deny',+ },+ {+ label: 'set model',+ path: '/rpc/hosts/chatllm/model',+ body: { chatId: 'c1', model: 'claude-haiku-4-5' },+ upstreamPath: '/v1/runtime/model',+ },+] as const++function post(route: (typeof MUTATING_ROUTES)[number]) {+ return request(makeApp()).post(route.path).set('authorization', 'Bearer tok').send(route.body)+}++describe.each(MUTATING_ROUTES)('R2-M3 / R1-M12 $label route deadlines', route => {+ it('waits for a slow upstream past upstreamTimeoutMs and relays its real answer', async () => {+ config.upstreamTimeoutMs = 60+ await startUpstream((_req, res) => {+ setTimeout(() => {+ res.writeHead(200, { 'content-type': 'application/json' })+ res.end(JSON.stringify({ applied: true }))+ }, 400)+ })++ const startedAt = Date.now()+ const res = await post(route).expect(200)++ // Witness: the upstream received exactly this one request, was still working+ // on it well after the 60 ms deadline, and completed it (not closed early).+ expect(seen).toEqual([{ method: 'POST', url: route.upstreamPath, closedEarly: false }])+ expect(Date.now() - startedAt).toBeGreaterThanOrEqual(350)+ expect(JSON.parse(res.text)).toEqual({ applied: true })+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ })++ it('a wake retry that hangs ends in 504 by its own bounded timeout', async () => {+ config.upstreamTimeoutMs = 150+ await startUpstream((req, _res, index) => {+ if (index === 0) dropConnection(req) // first attempt: host down+ // second attempt (the wake retry): never answered+ })++ const startedAt = Date.now()+ const res = await post(route).expect(504)++ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ // Witness: two requests reached the upstream. The second is the retry; the+ // proxy aborted it (closedEarly) at ~upstreamTimeoutMs, not at some later+ // point, so the 504 comes from the retry deadline.+ expect(seen.map(entry => entry.url)).toEqual([route.upstreamPath, route.upstreamPath])+ await vi.waitFor(() => expect(seen[1]!.closedEarly).toBe(true))+ expect(Date.now() - startedAt).toBeLessThan(2_000)+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ })++ it('a wake retry is bounded by the remaining hold budget, not by upstreamTimeoutMs', async () => {+ config.upstreamTimeoutMs = 30_000+ config.wakeMaxHoldMs = 1_300+ await startUpstream((req, _res, index) => {+ if (index === 0) dropConnection(req)+ })++ const startedAt = Date.now()+ await post(route).expect(504)+ const elapsed = Date.now() - startedAt++ expect(seen).toHaveLength(2)+ await vi.waitFor(() => expect(seen[1]!.closedEarly).toBe(true))+ // The hold budget (1.3 s) ended it, far below the 30 s upstream timeout.+ expect(elapsed).toBeGreaterThanOrEqual(1_000)+ expect(elapsed).toBeLessThan(5_000)+ })++ // R1-M12: no idempotency key, so a duplicated POST is a duplicated side+ // effect. A host that keeps dropping connections must see a small, fixed+ // number of POSTs however long the hold window is.+ it('re-issues the POST at most once while the host keeps failing (4 s hold)', async () => {+ config.upstreamTimeoutMs = 2_000+ config.wakeMaxHoldMs = 4_000+ await startUpstream(req => dropConnection(req))++ const startedAt = Date.now()+ const res = await post(route).expect(503)++ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })+ // Witness: the host was reached (POSTs observed) and a wake was requested.+ expect(seen.length).toBeGreaterThanOrEqual(1)+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalled()+ expect(seen).toHaveLength(2)+ // It answered promptly instead of looping through the whole 4 s window.+ expect(Date.now() - startedAt).toBeLessThan(3_000)+ })++ // NEW-fa-2: a retry with less than MIN_ADMISSION_RETRY_TIMEOUT_MS (1 s) of hold+ // budget left would time out after the upstream applied it. The 500 ms hold+ // budget leaves ~490 ms once the first attempt has failed: inside the+ // 100-999 ms band a lowered floor would send a second POST.+ it('sends no wake retry when less than the 1 s admission floor of the hold budget remains', async () => {+ config.upstreamTimeoutMs = 30_000+ config.wakeMaxHoldMs = 500+ await startUpstream(req => dropConnection(req))++ const res = await post(route).expect(503)++ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })+ // Witness: the first attempt reached the upstream and the wake was requested,+ // so the single POST is the first attempt and the retry was withheld.+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(seen.map(entry => `${entry.method} ${entry.url}`)).toEqual([+ `POST ${route.upstreamPath}`,+ ])+ })+})++describe('R1-M12 /messages keeps the retry-until-deadline loop (it carries a messageId)', () => {+ it('re-issues the POST until the host admits the request', async () => {+ config.upstreamTimeoutMs = 2_000+ config.wakeMaxHoldMs = 4_000+ await startUpstream((req, res, index) => {+ if (index < 3) {+ dropConnection(req)+ return+ }+ res.writeHead(200, { 'content-type': 'application/json' })+ res.end(JSON.stringify({ success: true, taskId: 't-1' }))+ })++ await request(makeApp())+ .post('/rpc/hosts/chatllm/messages')+ .set('authorization', 'Bearer tok')+ .send({ content: 'hello' })+ .expect(200)++ // Three dropped connections then an accepted one: more than the single+ // re-issue every other route is limited to.+ expect(seen).toHaveLength(4)+ expect(seen.every(entry => entry.url.startsWith('/v1/runtime/'))).toBe(true)+ })++ // NEW-fa-2: even the retry-until-deadline route stops re-issuing once less than+ // the 1 s admission floor of the hold budget remains (100-999 ms band).+ it('sends no retry when less than the 1 s admission floor of the hold budget remains', async () => {+ config.upstreamTimeoutMs = 30_000+ config.wakeMaxHoldMs = 500+ await startUpstream(req => dropConnection(req))++ const res = await request(makeApp())+ .post('/rpc/hosts/chatllm/messages')+ .set('authorization', 'Bearer tok')+ .send({ content: 'hello' })+ .expect(503)++ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })+ // Witness: the first attempt reached the upstream and the wake was requested.+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(seen).toHaveLength(1)+ expect(seen[0]!.url.startsWith('/v1/runtime/')).toBe(true)+ })+})++describe('artifact download deadlines (NEW-rpx-1)', () => {+ const DOWNLOAD_PATH = '/rpc/hosts/chatllm/artifacts/report.bin/download'+ const UPSTREAM_DOWNLOAD_PATH = '/v1/runtime/artifacts/report.bin/download'++ function download() {+ return request(makeApp()).get(DOWNLOAD_PATH).set('authorization', 'Bearer tok')+ }++ it('slow response headers are cut off by upstreamTimeoutMs and answer 504', async () => {+ config.upstreamTimeoutMs = 30+ config.artifactDownloadTimeoutMs = 60_000+ await startUpstream((_req, res) => {+ // Headers would arrive after 1.5 s; the client must give up long before.+ setTimeout(() => {+ if (res.destroyed) return+ res.writeHead(200, { 'content-type': 'application/octet-stream' })+ res.end('late')+ }, 1_500)+ })++ const startedAt = Date.now()+ const res = await download().expect(504)++ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ expect(seen.map(entry => entry.url)).toEqual([UPSTREAM_DOWNLOAD_PATH])+ await vi.waitFor(() => expect(seen[0]!.closedEarly).toBe(true))+ expect(Date.now() - startedAt).toBeLessThan(1_200)+ })++ it('a body that stalls after the headers is cut off by artifactDownloadTimeoutMs and answers 504', async () => {+ config.upstreamTimeoutMs = 10_000+ config.artifactDownloadTimeoutMs = 50+ let headersFlushed = false+ await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': '1000' })+ res.write('first-bytes', () => {+ headersFlushed = true+ })+ // ...and then nothing: the body never completes.+ })++ const startedAt = Date.now()+ const res = await download().expect(504)++ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ // Witness: the upstream had already sent its headers and first bytes, so the+ // 504 is the body-phase timer, not the 10 s header timer.+ expect(seen.map(entry => entry.url)).toEqual([UPSTREAM_DOWNLOAD_PATH])+ expect(headersFlushed).toBe(true)+ await vi.waitFor(() => expect(seen[0]!.closedEarly).toBe(true))+ expect(Date.now() - startedAt).toBeLessThan(3_000)+ })++ // NEW-sec-4: on a wake retry the body phase gets min(artifactDownloadTimeoutMs,+ // remaining hold budget), so a stalled retry cannot outlive the request.+ it('a stalled body on the wake retry is bounded by the remaining hold budget', async () => {+ config.upstreamTimeoutMs = 10_000+ config.artifactDownloadTimeoutMs = 60_000+ config.wakeMaxHoldMs = 1_400+ await startUpstream((req, res, index) => {+ if (index === 0) {+ dropConnection(req)+ return+ }+ res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': '1000' })+ res.write('first-bytes')+ })++ const startedAt = Date.now()+ await download().expect(504)+ const elapsed = Date.now() - startedAt++ // Witness: the wake happened and the retry reached the upstream, which began+ // to answer (headers + first bytes) before stalling.+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(seen).toHaveLength(2)+ await vi.waitFor(() => expect(seen[1]!.closedEarly).toBe(true))+ // 1.4 s hold budget, not the 60 s download timeout.+ expect(elapsed).toBeLessThan(5_000)+ })++ // NEW-tq-3(c): both download timers are released once the response is+ // committed, so a completed download leaves no armed timer behind. Sentinel+ // delays (7777 header, 8888 body) identify the route's own timers among the+ // ones express/undici/supertest create.+ it('clears both the header timer and the body timer after a successful download', async () => {+ config.upstreamTimeoutMs = 7_777+ config.artifactDownloadTimeoutMs = 8_888+ const realSetTimeout = globalThis.setTimeout+ const realClearTimeout = globalThis.clearTimeout+ const armed = new Map<unknown, number>()+ const cleared = new Set<unknown>()+ const setSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((+ handler: () => void,+ delay?: number,+ ...args: unknown[]+ ) => {+ const timer = realSetTimeout(handler, delay, ...args)+ if (delay === 7_777 || delay === 8_888) armed.set(timer, delay)+ return timer+ }) as typeof setTimeout)+ const clearSpy = vi.spyOn(globalThis, 'clearTimeout').mockImplementation(((+ timer?: Parameters<typeof clearTimeout>[0]+ ) => {+ cleared.add(timer)+ return realClearTimeout(timer)+ }) as typeof clearTimeout)+ await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/octet-stream' })+ res.end('artifact-bytes')+ })++ try {+ await download().expect(200)+ } finally {+ setSpy.mockRestore()+ clearSpy.mockRestore()+ }++ // Witness: the route armed the header timer AND re-armed a body timer.+ expect([...armed.values()].sort()).toEqual([7_777, 8_888])+ for (const timer of armed.keys()) expect(cleared.has(timer)).toBe(true)+ })++ it('a healthy download still completes (control for the deadline tests)', async () => {+ config.upstreamTimeoutMs = 2_000+ config.artifactDownloadTimeoutMs = 2_000+ await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/octet-stream' })+ res.end('artifact-bytes')+ })++ const res = await download().expect(200)++ expect(Buffer.from(res.body).toString()).toBe('artifact-bytes')+ expect(seen).toHaveLength(1)+ })+})++describe('artifacts list keeps a finite first-attempt deadline', () => {+ it('returns 504 when the upstream never answers (read-only, small JSON)', async () => {+ config.upstreamTimeoutMs = 60+ await startUpstream(() => {+ // never answered+ })++ const res = await request(makeApp())+ .get('/rpc/hosts/chatllm/artifacts')+ .set('authorization', 'Bearer tok')+ .expect(504)++ expect(res.body).toEqual({ error: 'Gateway Timeout' })+ expect(seen.map(entry => entry.url)).toEqual(['/v1/runtime/artifacts'])+ await vi.waitFor(() => expect(seen[0]!.closedEarly).toBe(true))+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ })+})diff --git a/rpc-proxy/src/__tests__/session-rename-passthrough.test.ts b/rpc-proxy/src/__tests__/session-rename-passthrough.test.tsindex c1d28c348..eaf2e65b2 100644--- a/rpc-proxy/src/__tests__/session-rename-passthrough.test.ts+++ b/rpc-proxy/src/__tests__/session-rename-passthrough.test.ts@@ -120,7 +120,10 @@ describe('PATCH /rpc/hosts/:hostRef/sessions/:agent/:chatId/name — rename pass})it('returns 403 when the user cannot access the host', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const fetchMock = vi.fn()globalThis.fetch = fetchMock as unknown as typeof fetchdiff --git a/rpc-proxy/src/__tests__/sessions-passthrough.test.ts b/rpc-proxy/src/__tests__/sessions-passthrough.test.tsindex 21ce230b3..4438be2ec 100644--- a/rpc-proxy/src/__tests__/sessions-passthrough.test.ts+++ b/rpc-proxy/src/__tests__/sessions-passthrough.test.ts@@ -162,7 +162,10 @@ describe('GET /rpc/hosts/:hostRef/sessions — passthrough to mcp-host', () => {})it('returns 403 if the user cannot access the host', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })await request(makeApp()).get('/rpc/hosts/other-host/sessions').set('authorization', 'Bearer user-token')@@ -422,7 +425,10 @@ describe('GET /rpc/hosts/:hostRef/sessions/:agent/:chatId/context-breakdown —})it('returns 403 if the user cannot access the host', async () => {- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })await request(makeApp()).get('/rpc/hosts/other-host/sessions/chatllm/c1/context-breakdown').set('authorization', 'Bearer user-token')diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.tsindex 3a2280d7b..08fea4551 100644--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts@@ -4,6 +4,7 @@ import { createServer } from 'node:http'import type { AddressInfo } from 'node:net'import request from 'supertest'import { config } from '../config.js'+import { apiErrorHandler } from '../errorHandler.js'import { createRpcRouter } from '../routes/rpc.js'import { forwardHostMessageToHost } from '../services/mcpHostRestService.js'@@ -73,11 +74,7 @@ function makeApp() {const app = express()app.use(express.json())app.use(createRpcRouter())- app.use(- (err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {- res.status(500).json({ error: err instanceof Error ? err.message : 'Internal error' })- }- )+ app.use(apiErrorHandler)return app}diff --git a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts b/rpc-proxy/src/__tests__/wake-route-coverage.test.tsindex 5cee76bd8..7643ebdb5 100644--- a/rpc-proxy/src/__tests__/wake-route-coverage.test.ts+++ b/rpc-proxy/src/__tests__/wake-route-coverage.test.ts@@ -3,12 +3,12 @@ import type { Response as ExpressResponse } from 'express'import express from 'express'import request from 'supertest'import { config } from '../config.js'+import { apiErrorHandler } from '../errorHandler.js'import {createRpcRouter,respondControlApiHostAccessRejection,respondUpstreamUnavailable,} from '../routes/rpc.js'-import { isUpstreamTimeoutError } from '../services/wakeAndHold.js'// ── Issue #791 §11.4/§11.5 — extend wake-and-hold to the remaining finite// Desktop routes, and prove terminal-response (duplicate-write) safety in the@@ -72,15 +72,7 @@ function makeApp() {const app = express()app.use(express.json())app.use(createRpcRouter())- app.use(- (err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {- if (isUpstreamTimeoutError(err)) {- res.status(504).json({ error: 'Gateway Timeout' })- return- }- res.status(500).json({ error: err instanceof Error ? err.message : 'Internal error' })- }- )+ app.use(apiErrorHandler)return app}@@ -125,7 +117,12 @@ afterEach(() => {config.upstreamTimeoutMs = originalUpstreamTimeoutMs})-describe('finite route upstream deadlines', () => {+// R2-M3: deny and set-model mutate host state, so their FIRST attempt carries no+// client-side deadline (a timeout would 504 a change the host may have applied).+// Only the read-only artifacts list keeps a finite first-attempt deadline. The+// real-socket versions of these (slow upstream, hung wake retry) are in+// route-upstream-deadlines.real-http.test.ts.+describe('first-attempt upstream deadlines', () => {it.each([{label: 'deny approval',@@ -139,15 +136,28 @@ describe('finite route upstream deadlines', () => {path: '/rpc/hosts/chatllm/model',body: { chatId: 'c1', model: 'claude-haiku-4-5' },},- {- label: 'list artifacts',- scope: 'host:task:read',- path: '/rpc/hosts/chatllm/artifacts',- body: null,- },- ])('$label returns 504 when the actual upstream deadline aborts a hung fetch', async row => {+ ])('$label sends its first attempt without an abort signal', async row => {config.upstreamTimeoutMs = 30authTokenMock.verifyRpcToken.mockReturnValue(claims([row.scope, 'host:wake:write']))+ const fetchMock = vi+ .fn()+ .mockResolvedValue(mockFetchResponse(200, JSON.stringify({ ok: true })))+ globalThis.fetch = fetchMock as unknown as typeof fetch++ await request(makeApp())+ .post(row.path)+ .send(row.body)+ .set('authorization', 'Bearer tok')+ .expect(200)++ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect((fetchMock.mock.calls[0]![1] as RequestInit).signal).toBeUndefined()+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ })++ it('list artifacts returns 504 when the upstream deadline aborts a hung fetch', async () => {+ config.upstreamTimeoutMs = 30+ authTokenMock.verifyRpcToken.mockReturnValue(claims(['host:task:read', 'host:wake:write']))let upstreamSignal: AbortSignal | undefinedconst fetchMock = vi.fn().mockImplementation((_url: string, init: RequestInit) => {upstreamSignal = init.signal as AbortSignal@@ -159,11 +169,10 @@ describe('finite route upstream deadlines', () => {})globalThis.fetch = fetchMock as unknown as typeof fetch- const app = makeApp()- const outgoing = row.body- ? request(app).post(row.path).send(row.body)- : request(app).get(row.path)- const res = await outgoing.set('authorization', 'Bearer tok').expect(504)+ const res = await request(makeApp())+ .get('/rpc/hosts/chatllm/artifacts')+ .set('authorization', 'Bearer tok')+ .expect(504)expect(res.body).toEqual({ error: 'Gateway Timeout' })expect(upstreamSignal?.aborted).toBe(true)@@ -607,8 +616,24 @@ describe('§11.5 route outer-catch responders guard headersSent (one-shot settleit('respondControlApiHostAccessRejection is a no-op once the response is committed', () => {const res = makeRes(true)expect(() =>- respondControlApiHostAccessRejection(res as unknown as ExpressResponse, 403)+ respondControlApiHostAccessRejection(res as unknown as ExpressResponse, {+ status: 403,+ code: 'host_access_revoked',+ })).not.toThrow()expect(res.statusCode).toBe(0)})++ it('respondControlApiHostAccessRejection writes the 403 with its denial code when uncommitted', () => {+ const res = makeRes(false)+ respondControlApiHostAccessRejection(res as unknown as ExpressResponse, {+ status: 403,+ code: 'host_access_revoked',+ })+ expect(res.statusCode).toBe(403)+ expect(res.body).toEqual({+ error: 'Forbidden: user cannot access this host',+ code: 'host_access_revoked',+ })+ })})diff --git a/rpc-proxy/src/app.ts b/rpc-proxy/src/app.tsindex 4c33fbc66..443a7e682 100644--- a/rpc-proxy/src/app.ts+++ b/rpc-proxy/src/app.ts@@ -1,6 +1,7 @@-import express, { NextFunction, Request, Response } from 'express'+import express from 'express'import cors from 'cors'import { config } from './config.js'+import { apiErrorHandler } from './errorHandler.js'import { createDesktopRouter } from './routes/desktopProxy.js'import { createHealthRouter } from './routes/health.js'import { createMcpOauthRouter } from './routes/mcpOauth.js'@@ -9,21 +10,6 @@ import { createRpcHostActivityStreamRouter } from './routes/rpcHostActivityStreaimport { createRpcHostProgressStreamRouter } from './routes/rpcHostProgressStream.js'import { createRpcHostStatusStreamRouter } from './routes/rpcHostStatusStream.js'import { createSandboxUiSessionRouter } from './routes/sandboxUi.js'-import { isUpstreamTimeoutError } from './services/wakeAndHold.js'--/**- * body-parser flags a request whose declared/streamed body crossed the parser's- * `limit` with `type: 'entity.too.large'`. The terminal error handler below- * would otherwise report it as a 500, which reads like a server fault for what- * is a client-sized payload; mcp-host answers the same condition with 413.- */-function isEntityTooLargeError(error: unknown): boolean {- return (- typeof error === 'object' &&- error !== null &&- (error as { type?: unknown }).type === 'entity.too.large'- )-}export function createApp() {const app = express()@@ -51,22 +37,7 @@ export function createApp() {res.status(404).json({ error: 'Not Found' })})- app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => {- if (isUpstreamTimeoutError(err)) {- res.status(504).json({ error: 'Gateway Timeout' })- return- }-- if (isEntityTooLargeError(err)) {- res.status(413).json({ error: 'Payload Too Large' })- return- }-- res.status(500).json({- error: 'Internal Server Error',- message: err instanceof Error ? err.message : 'Unknown error',- })- })+ app.use(apiErrorHandler)return app}diff --git a/rpc-proxy/src/config.ts b/rpc-proxy/src/config.tsindex 0f9c377b8..1a2bc84fe 100644--- a/rpc-proxy/src/config.ts+++ b/rpc-proxy/src/config.ts@@ -29,7 +29,8 @@ type Config = {/*** Stateless wake-and-hold (Stage 5): a request hitting a down/draining* stateless host is parked while control-api wakes the pod, bounded by- * `wakeMaxHoldMs`. Readiness is re-checked every `wakePollMs` and the wake+ * `wakeMaxHoldMs` (values above MAX_REQUEST_HOLD_MS are clamped at load).+ * Readiness is re-checked every `wakePollMs` and the wake* is re-triggered every `wakeRetriggerMs` (covers a dropped HCC watch* event). See src/services/wakeAndHold.ts.*/@@ -136,15 +137,43 @@ export function parseArtifactDownloadMaxBytes(rawMb: string): number {return bytes}+/** Largest delay setTimeout/AbortSignal.timeout accept; above it they fire immediately. */+export const MAX_TIMER_DELAY_MS = 2_147_483_647+export function parsePositiveIntMs(name: string, raw: string): number {const trimmed = raw.trim()const n = Number(trimmed)if (!Number.isInteger(n) || n < 1) {throw new Error(`${name} contains invalid value "${trimmed}" (must be a positive integer)`)}+ if (n > MAX_TIMER_DELAY_MS) {+ throw new Error(+ `${name} contains invalid value "${trimmed}" (must be at most ${MAX_TIMER_DELAY_MS} ms)`+ )+ }return n}+/**+ * Upper bound of one held request: it leaves room for a final upstream attempt+ * before Desktop's own 60s deadline. The hold budget is capped here, at config+ * load, so the configured value is the effective one.+ */+export const MAX_REQUEST_HOLD_MS = 48_000++/**+ * An operator-set value above the cap is clamped and warned about at boot.+ * The built-in default equals the cap, so it never reaches the warning.+ */+export function parseWakeMaxHoldMs(raw: string): number {+ const configuredMs = parsePositiveIntMs('RPC_PROXY_WAKE_MAX_HOLD_MS', raw)+ if (configuredMs <= MAX_REQUEST_HOLD_MS) return configuredMs+ console.warn(+ `[RPC_PROXY] RPC_PROXY_WAKE_MAX_HOLD_MS=${configuredMs} exceeds the request hold cap; clamped to ${MAX_REQUEST_HOLD_MS}`+ )+ return MAX_REQUEST_HOLD_MS+}+const DEV_RPC_JWT_PUBLIC_KEY = normalizePem(`-----BEGIN PUBLIC KEY-----MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArCIYGHehMPpGKePxaKQarDX5yrzifU5i4fzpI3EtkKSU6s5ug7EkKxc2DdMekoqXe9vr7qKyVwiilUIusXLX@@ -165,7 +194,10 @@ export const config: Config = {),jwtIssuer: requiredOrDevDefault('RPC_PROXY_JWT_ISSUER', 'control-api'),jwtAudience: requiredOrDevDefault('RPC_PROXY_JWT_AUDIENCE', 'rpc-proxy'),- upstreamTimeoutMs: Number(process.env.RPC_PROXY_UPSTREAM_TIMEOUT_MS || 8000),+ upstreamTimeoutMs: parsePositiveIntMs(+ 'RPC_PROXY_UPSTREAM_TIMEOUT_MS',+ process.env.RPC_PROXY_UPSTREAM_TIMEOUT_MS || '8000'+ ),maxTokenLength: Number(process.env.RPC_PROXY_MAX_TOKEN_LENGTH || 4096),allowedMethodPattern: new RegExp(process.env.RPC_PROXY_ALLOWED_METHOD_PATTERN || '^[a-zA-Z0-9_./:-]{1,120}$'@@ -207,10 +239,7 @@ export const config: Config = {activityStreamMaxPerUserHost: Number(process.env.RPC_PROXY_ACTIVITY_STREAM_MAX_PER_USER_HOST || 1),- wakeMaxHoldMs: parsePositiveIntMs(- 'RPC_PROXY_WAKE_MAX_HOLD_MS',- process.env.RPC_PROXY_WAKE_MAX_HOLD_MS || '90000'- ),+ wakeMaxHoldMs: parseWakeMaxHoldMs(process.env.RPC_PROXY_WAKE_MAX_HOLD_MS || '48000'),wakePollMs: parsePositiveIntMs('RPC_PROXY_WAKE_POLL_MS',process.env.RPC_PROXY_WAKE_POLL_MS || '2000'diff --git a/rpc-proxy/src/errorHandler.ts b/rpc-proxy/src/errorHandler.tsnew file mode 100644index 000000000..20e19f2e1--- /dev/null+++ b/rpc-proxy/src/errorHandler.ts@@ -0,0 +1,43 @@+import type { NextFunction, Request, Response } from 'express'+import { isUpstreamTimeoutError } from './services/wakeAndHold.js'++/**+ * body-parser flags a request whose declared/streamed body crossed the parser's+ * `limit` with `type: 'entity.too.large'`. The terminal error handler below+ * would otherwise report it as a 500, which reads like a server fault for what+ * is a client-sized payload; mcp-host answers the same condition with 413.+ */+function isEntityTooLargeError(error: unknown): boolean {+ return (+ typeof error === 'object' &&+ error !== null &&+ (error as { type?: unknown }).type === 'entity.too.large'+ )+}++/**+ * The app's terminal Express error handler. It lives in its own module so route+ * tests mount this very function instead of a copy of its 504 mapping: a test+ * app that re-implements the mapping keeps passing after production changes it.+ */+export function apiErrorHandler(+ err: unknown,+ _req: Request,+ res: Response,+ _next: NextFunction+): void {+ if (isUpstreamTimeoutError(err)) {+ res.status(504).json({ error: 'Gateway Timeout' })+ return+ }++ if (isEntityTooLargeError(err)) {+ res.status(413).json({ error: 'Payload Too Large' })+ return+ }++ res.status(500).json({+ error: 'Internal Server Error',+ message: err instanceof Error ? err.message : 'Unknown error',+ })+}diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.tsindex 09d2bfccd..d96cc5b5a 100644--- a/rpc-proxy/src/routes/rpc.ts+++ b/rpc-proxy/src/routes/rpc.ts@@ -17,6 +17,11 @@ import {fetchUserConnectorsFromControlApi,requestHostWakeFromControlApi,} from '../services/controlApiRestService.js'+import {+ type HostAccessDenialCode,+ isHostAccessDenied,+ respondHostAccessDenied,+} from '../services/hostAccessDenial.js'import {type HostRuntimeMessageRequest,forwardCancelToHost,@@ -32,7 +37,6 @@ import {validateRpcRequest,} from '../services/mcpProxyService.js'import {- MAX_REQUEST_HOLD_MS,isUpstreamTimeoutError,isWakeEligibleHostError,respondWithWakeAndHold,@@ -124,35 +128,57 @@ function sessionDrainingFence(response: Response, body: string): Error | null {})}-function controlApiHostAccessRejectionStatus(error: unknown): 401 | 403 | 409 | null {+export type ControlApiHostAccessRejection =+ | { status: 401 | 409 }+ | { status: 403; code: HostAccessDenialCode }++function controlApiHostAccessRejection(error: unknown): ControlApiHostAccessRejection | null {if (!(error instanceof Error) || error.name !== 'ControlApiHostAccessRejectedError') {return null}- const status = (error as Error & { status?: unknown }).status- return status === 401 || status === 403 || status === 409 ? status : null+ const { status, denialCode } = error as Error & { status?: unknown; denialCode?: unknown }+ if (status === 401 || status === 409) return { status }+ if (+ status === 403 &&+ (denialCode === 'host_access_revoked' || denialCode === 'host_access_denied')+ ) {+ return { status, code: denialCode }+ }+ return null}export function respondControlApiHostAccessRejection(res: ExpressResponse,- status: 401 | 403 | 409+ rejection: ControlApiHostAccessRejection): void {if (res.headersSent) {console.warn(- `[RPC_PROXY] suppressing duplicate terminal response (host-access-rejection ${status})`+ `[RPC_PROXY] suppressing duplicate terminal response (host-access-rejection ${rejection.status})`)return}- if (status === 401) {+ if (rejection.status === 401) {res.status(401).json({ error: 'Unauthorized' })return}- if (status === 403) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (rejection.status === 403) {+ respondHostAccessDenied(res, rejection.code)return}res.status(409).json({ error: 'Direct run attribution conflict' })}+/**+ * Abort signal for a MUTATING upstream call (approve, deny, set-model). The+ * first attempt has no client-side timeout: aborting a request the upstream may+ * already have applied would answer 504 for a change that took effect, and the+ * user's retry would then hit "No pending approval". Only a wake-and-hold retry+ * passes `timeoutMs`, bounded by the hold deadline.+ */+function mutatingCallSignal(timeoutMs: number | undefined): AbortSignal | undefined {+ return timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs)+}+class ProxyArtifactTooLargeError extends Error {constructor(readonly maxBytes: number) {super(`Artifact too large to download: max ${maxBytes} bytes`)@@ -404,7 +430,7 @@ export function createRpcRouter(): Router {: {}),}- const wakeDeadlineMs = Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)+ const wakeDeadlineMs = Date.now() + config.wakeMaxHoldMsconst host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,...(traceContext.sessionId@@ -417,8 +443,8 @@ export function createRpcRouter(): Router {}: {}),})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}if (host.attributionBindingStatus === 'unavailable') {@@ -465,6 +491,9 @@ export function createRpcRouter(): Router {claims: auth,rpcAccessToken,deadlineMs: wakeDeadlineMs,+ // The per-request messageId makes a re-presented POST a+ // duplicate_delivery replay, so retrying until the deadline is safe.+ retryUntilDeadline: true,attemptUpstream: async timeoutMs => {const retried = await forwardHostMessageToHost(host, forwardedBody, {async: isAsync,@@ -485,9 +514,9 @@ export function createRpcRouter(): Router {// the wake path (which would re-forward an already-delivered message).res.status(200).json(upstreamResponse)} catch (error) {- const rejectedStatus = controlApiHostAccessRejectionStatus(error)- if (rejectedStatus) {- respondControlApiHostAccessRejection(res, rejectedStatus)+ const rejection = controlApiHostAccessRejection(error)+ if (rejection) {+ respondControlApiHostAccessRejection(res, rejection)return}console.warn(@@ -523,8 +552,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}@@ -607,8 +636,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -625,12 +654,12 @@ export function createRpcRouter(): Router {// host:approval:write; wake capability rides on the token. A suspended// (network-down) or draining Host triggers a wake-and-hold instead of a// bare next(error)/passthrough.- const attempt = async (timeoutMs = config.upstreamTimeoutMs) => {+ const attempt = async (timeoutMs?: number) => {const response = await fetch(`${baseUrl}/v1/runtime/approvals/approve`, {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(upstreamBody),- signal: AbortSignal.timeout(timeoutMs),+ signal: mutatingCallSignal(timeoutMs),})const body = await response.text()const draining = sessionDrainingFence(response, body)@@ -675,8 +704,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -689,12 +718,12 @@ export function createRpcRouter(): Router {requestId: parsed.toolCallId || parsed.requestId,}// Wake-eligible finite operation (§11.4): scope stays host:approval:write.- const attempt = async (timeoutMs = config.upstreamTimeoutMs) => {+ const attempt = async (timeoutMs?: number) => {const response = await fetch(`${baseUrl}/v1/runtime/approvals/deny`, {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(upstreamBody),- signal: AbortSignal.timeout(timeoutMs),+ signal: mutatingCallSignal(timeoutMs),})const body = await response.text()const draining = sessionDrainingFence(response, body)@@ -757,8 +786,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -855,8 +884,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -938,8 +967,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -1019,8 +1048,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -1085,8 +1114,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')@@ -1150,8 +1179,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const body = req.body as Record<string, unknown>@@ -1162,12 +1191,12 @@ export function createRpcRouter(): Router {const baseUrl = host.url.replace(/\/+$/, '')console.info(`[RPC_PROXY] user=${auth.sub} host=${hostRef} method=set-model`)// Wake-eligible finite operation (§11.4): scope stays host:model:write.- const attempt = async (timeoutMs = config.upstreamTimeoutMs) => {+ const attempt = async (timeoutMs?: number) => {const response = await fetch(`${baseUrl}/v1/runtime/model`, {method: 'POST',headers: { 'content-type': 'application/json', ...host.headers },body: JSON.stringify(body),- signal: AbortSignal.timeout(timeoutMs),+ signal: mutatingCallSignal(timeoutMs),})const upstreamBody = await response.text()const draining = sessionDrainingFence(response, upstreamBody)@@ -1219,7 +1248,7 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {+ if (isHostAccessDenied(host)) {res.status(404).json({ error: 'Host not found or not accessible' })return}@@ -1286,7 +1315,7 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {+ if (isHostAccessDenied(host)) {res.status(404).json({ error: 'Host not found or not accessible' })return}@@ -1356,12 +1385,16 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')// Wake-eligible finite operation (§11.4): scope stays host:task:read.+ // Unlike approve/deny/model this is a read-only list of small JSON: a+ // timeout cannot leave a half-applied change, and without one a hung+ // host would pin this socket. It therefore keeps a finite deadline on+ // the first attempt as well.const attempt = async (timeoutMs = config.upstreamTimeoutMs) => {const response = await fetch(`${baseUrl}/v1/runtime/artifacts`, {headers: { ...host.headers },@@ -1424,15 +1457,18 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const baseUrl = host.url.replace(/\/+$/, '')// Wake-eligible finite operation (§11.4): scope stays host:task:read.// The success path only commits (`res.send`) at the very end, so a// wake retry before that point is safe from duplicate delivery.- const attemptDownload = async (timeoutMs = config.upstreamTimeoutMs) => {+ const attemptDownload = async (+ timeoutMs = config.upstreamTimeoutMs,+ holdDeadlineMs?: number+ ) => {const controller = new AbortController()let timeout = setTimeout(() => controller.abort(), timeoutMs)try {@@ -1446,9 +1482,19 @@ export function createRpcRouter(): Router {// The upstream timeout protects connection and header latency. Once// headers arrive, allow the separately bounded artifact transfer to// finish; the small JSON-operation deadline must not truncate a- // valid large download while its body is streaming.+ // valid large download while its body is streaming. On a wake retry+ // the transfer is also bounded by what is left of the hold budget,+ // so a stalled body cannot outlive the request deadline.clearTimeout(timeout)- timeout = setTimeout(() => controller.abort(), config.artifactDownloadTimeoutMs)+ timeout = setTimeout(+ () => controller.abort(),+ holdDeadlineMs === undefined+ ? config.artifactDownloadTimeoutMs+ : Math.min(+ config.artifactDownloadTimeoutMs,+ Math.max(0, holdDeadlineMs - Date.now())+ )+ )if (!response.ok) {const errBody = await response.text()const draining = sessionDrainingFence(response, errBody)@@ -1514,8 +1560,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}const limit = Number(req.query.limit || 50)@@ -1552,8 +1598,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}@@ -1593,8 +1639,8 @@ export function createRpcRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}diff --git a/rpc-proxy/src/routes/rpcHostActivityStream.ts b/rpc-proxy/src/routes/rpcHostActivityStream.tsindex ce1264b05..abbb25f6c 100644--- a/rpc-proxy/src/routes/rpcHostActivityStream.ts+++ b/rpc-proxy/src/routes/rpcHostActivityStream.ts@@ -6,6 +6,7 @@ import {requireRpcAuth,requireScope,} from '../middleware/auth.js'+import { isHostAccessDenied, respondHostAccessDenied } from '../services/hostAccessDenial.js'import { resolveHostConnectionForUser } from '../services/mcpProxyService.js'const activeStreamCountsByUser = new Map<string, number>()@@ -99,8 +100,8 @@ export function createRpcHostActivityStreamRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}diff --git a/rpc-proxy/src/routes/rpcHostProgressStream.ts b/rpc-proxy/src/routes/rpcHostProgressStream.tsindex 6a22da640..d0d5ca150 100644--- a/rpc-proxy/src/routes/rpcHostProgressStream.ts+++ b/rpc-proxy/src/routes/rpcHostProgressStream.ts@@ -6,6 +6,7 @@ import {requireRpcAuth,requireScope,} from '../middleware/auth.js'+import { isHostAccessDenied, respondHostAccessDenied } from '../services/hostAccessDenial.js'import { resolveHostConnectionForUser } from '../services/mcpProxyService.js'function isWildcardOrInvalidHostRef(hostRef: string): boolean {@@ -72,8 +73,8 @@ export function createRpcHostProgressStreamRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}diff --git a/rpc-proxy/src/routes/rpcHostStatusStream.ts b/rpc-proxy/src/routes/rpcHostStatusStream.tsindex d3fd70544..335242542 100644--- a/rpc-proxy/src/routes/rpcHostStatusStream.ts+++ b/rpc-proxy/src/routes/rpcHostStatusStream.ts@@ -6,6 +6,7 @@ import {requireRpcAuth,requireScope,} from '../middleware/auth.js'+import { isHostAccessDenied, respondHostAccessDenied } from '../services/hostAccessDenial.js'import {UpstreamHostError,forwardHostStatus,@@ -95,8 +96,8 @@ export function createRpcHostStatusStreamRouter(): Router {const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,})- if (!host) {- res.status(403).json({ error: 'Forbidden: user cannot access this host' })+ if (isHostAccessDenied(host)) {+ respondHostAccessDenied(res, host)return}diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.tsindex 007bd6d06..59157fa72 100644--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts@@ -1,8 +1,15 @@import { describe, expect, it, vi } from 'vitest'+import { config } from '../../config.js'import {+ forwardCancelToHost,+ forwardHostActivity,+ forwardHostHealth,forwardHostMessageToHost,+ forwardHostStatus,+ forwardTaskResultFromHost,__test__normalizeHostStatusPayload as normalize,} from '../mcpHostRestService.js'+import { isUpstreamTimeoutError } from '../wakeAndHold.js'describe('host message availability probe', () => {it('aborts at the requested short timeout with the standard abort reason', async () => {@@ -32,6 +39,97 @@ describe('host message availability probe', () => {})})+// Every upstream forwarder aborts its own fetch when the deadline fires. What+// the caller sees is whatever `signal.reason` is: the app's 504 mapping and the+// wake coordinator both classify by error NAME (AbortError / TimeoutError), so+// an abort that carries a custom reason would be reported as a 500 / 502.+describe('upstream forwarders abort with the standard classified reason', () => {+ const HOST = { name: 'chatllm', url: 'http://chatllm:8080', headers: {} }++ const PRODUCERS: Array<{+ label: string+ timeoutMs: number+ call: () => Promise<unknown>+ }> = [+ {+ label: 'forwardHostMessageToHost',+ timeoutMs: 750,+ call: () =>+ forwardHostMessageToHost(+ HOST,+ { content: 'hello', hostRef: 'chatllm' },+ { timeoutMs: 750 }+ ),+ },+ {+ label: 'forwardTaskResultFromHost',+ timeoutMs: 640,+ call: () => forwardTaskResultFromHost(HOST, 'task-1', 640),+ },+ {+ label: 'forwardCancelToHost',+ timeoutMs: 530,+ call: () => forwardCancelToHost(HOST, 'task-1', 'user-1', 530),+ },+ {+ label: 'forwardHostStatus',+ timeoutMs: config.upstreamTimeoutMs,+ call: () => forwardHostStatus(HOST),+ },+ {+ label: 'forwardHostHealth',+ timeoutMs: config.upstreamTimeoutMs,+ call: () => forwardHostHealth(HOST),+ },+ {+ label: 'forwardHostActivity',+ timeoutMs: config.upstreamTimeoutMs,+ call: () => forwardHostActivity(HOST, 10),+ },+ ]++ it.each(PRODUCERS)(+ '$label rejects with an AbortError at its deadline, not earlier',+ async ({ timeoutMs, call }) => {+ vi.useFakeTimers()+ let signal: AbortSignal | undefined+ const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(+ (_input, init) =>+ new Promise((_resolve, reject) => {+ signal = init?.signal as AbortSignal+ signal.addEventListener('abort', () => reject(signal!.reason))+ })+ )+ try {+ const pending = call()+ const rejection = pending.then(+ () => {+ throw new Error('expected the forwarder to reject at its deadline')+ },+ (error: unknown) => error+ )++ // Liveness witness: the request is in flight and still un-aborted just+ // before the deadline, so the rejection below is the deadline timer.+ await vi.advanceTimersByTimeAsync(timeoutMs - 1)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(signal?.aborted).toBe(false)++ await vi.advanceTimersByTimeAsync(1)+ const error = await rejection++ expect(signal?.aborted).toBe(true)+ expect(error).toBeInstanceOf(Error)+ expect((error as Error).name).toBe('AbortError')+ expect(isUpstreamTimeoutError(error)).toBe(true)+ } finally {+ fetchMock.mockRestore()+ vi.useRealTimers()+ }+ }+ )+})+const baseUpstream = {agent: {state: 'idle',diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.tsindex 29265d94e..654fbfa98 100644--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts@@ -475,11 +475,12 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>function respondOptions(coordinator: WakeAndHoldCoordinator,res: FakeRes,- attemptUpstream: (timeoutMs: number) => Promise<void>,+ attemptUpstream: (timeoutMs: number, deadlineMs: number) => Promise<void>,overrides?: {hostRef?: stringhost?: ResolvedServerConnectionrespondLegacy?: (error: unknown) => void+ retryUntilDeadline?: boolean}) {const hostRef = overrides?.hostRef ?? 'chatllm'@@ -490,6 +491,7 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>claims: holdClaims(hostRef, Date.now() + 3_600_000),rpcAccessToken: 'rpc-token',attemptUpstream,+ retryUntilDeadline: overrides?.retryUntilDeadline,respondLegacy:overrides?.respondLegacy ??((_error: unknown) => {@@ -572,7 +574,9 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>.mockImplementationOnce(async () => {res.status(200).json({ success: true, taskId: 't-3' })})- const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))+ const pending = respondWithWakeAndHold(+ respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })+ )await vi.advanceTimersByTimeAsync(750) // 0ms + 250ms + 750ms attemptsawait pending@@ -591,7 +595,9 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>if (!upstreamAdmits) throw upstreamDrainingError()res.status(200).json({ success: true, taskId: 't-late-admission' })})- const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))+ const pending = respondWithWakeAndHold(+ respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })+ )await vi.advanceTimersByTimeAsync(1_250)expect(attemptUpstream).toHaveBeenCalledTimes(3)@@ -614,11 +620,13 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>const res = makeRes()const attemptUpstream = vi.fn().mockRejectedValue(upstreamDrainingError())const pending = respondWithWakeAndHold({- ...respondOptions(coordinator, res, attemptUpstream),- deadlineMs: Date.now() + 1_000,+ ...respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true }),+ deadlineMs: Date.now() + 2_500,})- await vi.advanceTimersByTimeAsync(1_000)+ // Attempts at 0, 250 and 750 ms; at 1750 ms only 750 ms remain, below the+ // admission floor, so no fourth attempt is made.+ await vi.advanceTimersByTimeAsync(2_500)await pendingexpect(attemptUpstream).toHaveBeenCalledTimes(3)expect(res.statusCode).toBe(503)@@ -635,14 +643,14 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>throw upstreamDrainingError()})const pending = respondWithWakeAndHold({- ...respondOptions(coordinator, res, attemptUpstream),- deadlineMs: Date.now() + 500,+ ...respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true }),+ deadlineMs: Date.now() + 1_500,})- await vi.advanceTimersByTimeAsync(500)+ await vi.advanceTimersByTimeAsync(1_500)await pending- expect(attemptUpstream.mock.calls.map(([timeoutMs]) => timeoutMs)).toEqual([500, 250])+ expect(attemptUpstream.mock.calls.map(([timeoutMs]) => timeoutMs)).toEqual([1_500, 1_250])expect(res.statusCode).toBe(503)})@@ -653,31 +661,104 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>const attemptUpstream = vi.fn(async (_timeoutMs: number) => {res.status(200).json({ success: true, taskId: 't-capped' })})+ const deadlineMs = Date.now() + config.upstreamTimeoutMs + 5_000const pending = respondWithWakeAndHold({...respondOptions(coordinator, res, attemptUpstream),- deadlineMs: Date.now() + config.upstreamTimeoutMs + 5_000,+ deadlineMs,})await pending- expect(attemptUpstream).toHaveBeenCalledWith(config.upstreamTimeoutMs)+ expect(attemptUpstream).toHaveBeenCalledWith(config.upstreamTimeoutMs, deadlineMs)expect(res.statusCode).toBe(200)})- it('skips an admission retry when less than 100 ms remains', async () => {+ // R1-M13: the retry floor sat at 100 ms, below the upstream's admission+ // latency, so a retry with a sliver of budget timed out AFTER the upstream+ // applied the request (applied-once + 504). Below the floor the caller gets a+ // retryable 503 instead.+ it('skips an admission retry when less than the 1 s floor remains', async () => {const { coordinator, requestWake } = makeCoordinator()requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })const res = makeRes()const attemptUpstream = vi.fn(async (_timeoutMs: number) => {})const pending = respondWithWakeAndHold({...respondOptions(coordinator, res, attemptUpstream),- deadlineMs: Date.now() + 99,+ deadlineMs: Date.now() + 999,})await pendingexpect(attemptUpstream).not.toHaveBeenCalled()expect(res.statusCode).toBe(503)+ expect(res.body).toMatchObject({ code: 'host_waking' })+ })++ it('lets a deliberately short upstream timeout retry below the 1 s floor', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })+ const res = makeRes()+ const attemptUpstream = vi.fn(async (_timeoutMs: number) => {+ res.status(200).json({ ok: true })+ })+ const previous = config.upstreamTimeoutMs+ config.upstreamTimeoutMs = 300+ try {+ const pending = respondWithWakeAndHold({+ ...respondOptions(coordinator, res, attemptUpstream),+ deadlineMs: Date.now() + 400,+ })+ await pending+ } finally {+ config.upstreamTimeoutMs = previous+ }++ expect(attemptUpstream).toHaveBeenCalledTimes(1)+ expect(attemptUpstream.mock.calls[0]![0]).toBe(300)+ expect(res.statusCode).toBe(200)+ })++ // R1-M12: without an idempotency key every extra POST is a duplicate side+ // effect, so a route that does not opt in gets ONE retry after the wake.+ it('re-issues a non-idempotent request at most once, then answers host_waking', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })+ const res = makeRes()+ const attemptUpstream = vi.fn(async (_timeoutMs: number) => {+ throw upstreamDrainingError()+ })+ const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))++ await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)+ await pending++ expect(attemptUpstream).toHaveBeenCalledTimes(1)+ expect(res.statusCode).toBe(503)+ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })+ await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)+ expect(attemptUpstream).toHaveBeenCalledTimes(1)+ })++ it('opting in with retryUntilDeadline keeps re-issuing until the host admits the request', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })+ const res = makeRes()+ const attemptUpstream = vi+ .fn()+ .mockRejectedValueOnce(upstreamDrainingError())+ .mockRejectedValueOnce(upstreamDrainingError())+ .mockImplementationOnce(async () => {+ res.status(200).json({ ok: true })+ })+ const pending = respondWithWakeAndHold(+ respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })+ )++ await vi.advanceTimersByTimeAsync(750)+ await pending++ expect(attemptUpstream).toHaveBeenCalledTimes(3)+ expect(res.statusCode).toBe(200)})it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {diff --git a/rpc-proxy/src/services/controlApiRestService.ts b/rpc-proxy/src/services/controlApiRestService.tsindex 65d772ac3..961d798f4 100644--- a/rpc-proxy/src/services/controlApiRestService.ts+++ b/rpc-proxy/src/services/controlApiRestService.ts@@ -1,5 +1,11 @@import { config } from '../config.js'import { ResolvedServerConnection } from '../types.js'+import {+ type HostAccessDenial,+ type HostAccessDenialCode,+ hostAccessDenialCodeForReason,+ hostAccessDenied,+} from './hostAccessDenial.js'export type UserAllowedServers = {userId: string@@ -42,9 +48,19 @@ export type DirectRunBindingRequest = {}export class ControlApiHostAccessRejectedError extends Error {- constructor(readonly status: number) {+ /**+ * Set for a 403 only: whether control-api's denial reason proves the access+ * was removed from the user. Absent for 401/409, which are not authorization+ * denials of the Host.+ */+ readonly denialCode: HostAccessDenialCode | null+ constructor(+ readonly status: number,+ denialCode: HostAccessDenialCode | null = null+ ) {super(`Control API rejected host access (${status})`)this.name = 'ControlApiHostAccessRejectedError'+ this.denialCode = denialCode}}@@ -207,7 +223,7 @@ export async function fetchHostConnectionFromControlApi(directRunBinding?: DirectRunBindingRequestfetchImpl?: typeof fetch} = {}-): Promise<ResolvedServerConnection | null> {+): Promise<ResolvedServerConnection | HostAccessDenial> {const directRunBinding = options.directRunBindingconst response = await (options.fetchImpl ?? fetch)(`${controlApiBaseUrl()}/rpc/access/users/${encodeURIComponent(userId)}/mcp-hosts/${encodeURIComponent(hostRef)}`,@@ -222,10 +238,17 @@ export async function fetchHostConnectionFromControlApi(})- if (!directRunBinding && (response.status === 403 || response.status === 404)) {- return null+ if (!directRunBinding && response.status === 403) {+ return hostAccessDenied(await readHostAccessDenialCode(response))}- if (response.status === 401 || response.status === 403 || response.status === 409) {+ if (!directRunBinding && response.status === 404) {+ await drainBody(response)+ return hostAccessDenied('host_access_denied')+ }+ if (response.status === 403) {+ throw new ControlApiHostAccessRejectedError(403, await readHostAccessDenialCode(response))+ }+ if (response.status === 401 || response.status === 409) {await drainBody(response)throw new ControlApiHostAccessRejectedError(response.status)}@@ -236,12 +259,12 @@ export async function fetchHostConnectionFromControlApi(const parsed = (await response.json()) as Partial<UserAllowedHost>if (parsed.userId !== userId) {- return null+ return hostAccessDenied('host_access_denied')}const url = typeof parsed.url === 'string' ? parsed.url.trim() : ''const resolvedHostRef = typeof parsed.hostRef === 'string' ? parsed.hostRef.trim() : ''if (!url || !resolvedHostRef || resolvedHostRef !== hostRef) {- return null+ return hostAccessDenied('host_access_denied')}const attributionBindingStatus = directRunBinding ? parsed.bindingStatus : undefinedif (@@ -277,6 +300,18 @@ export type HostWakeApiResponse =| { kind: 'rate-limited'; retryAfterSeconds: number }| { kind: 'auth'; status: number }+/**+ * control-api reports the denial reason in the `x-host-access-denial-reason`+ * response header; its 403 body is the fixed `{"error":"Forbidden"}` and is never+ * parsed. An absent or unknown header yields the non-revoking code: an+ * unreadable reason can never prove the access was removed.+ */+async function readHostAccessDenialCode(response: Response): Promise<HostAccessDenialCode> {+ const reason = response.headers.get('x-host-access-denial-reason')+ await drainBody(response)+ return hostAccessDenialCodeForReason(reason)+}+async function drainBody(response: Response): Promise<void> {try {await response.arrayBuffer()diff --git a/rpc-proxy/src/services/hostAccessDenial.ts b/rpc-proxy/src/services/hostAccessDenial.tsnew file mode 100644index 000000000..bbc2cdc4f--- /dev/null+++ b/rpc-proxy/src/services/hostAccessDenial.ts@@ -0,0 +1,53 @@+import type { Response as ExpressResponse } from 'express'++/**+ * rpc-proxy's OWN authorization denial for a Host (never a passthrough of an+ * mcp-host 403 body). Status stays 403 and `error` keeps its text; `code` is the+ * machine field Desktop uses to tell "access was removed from this user" from+ * "denied for a reason that does not prove revocation".+ */+export const HOST_ACCESS_DENIED_ERROR = 'Forbidden: user cannot access this host'++export type HostAccessDenialCode = 'host_access_revoked' | 'host_access_denied'++export type HostAccessDenial = { denied: true; code: HostAccessDenialCode }++/**+ * control-api denial reasons (control-api `RpcHostAccessDenialReason`) that+ * prove the access was actually removed from the user. Every other reason+ * (`host_disabled`, `host_missing`, `host_claim_missing`, an unknown or absent+ * reason) is a denial that does not prove revocation.+ */+const REVOKED_CONTROL_API_REASONS: ReadonlySet<string> = new Set([+ 'team_membership_missing',+ 'directory_grant_missing',+ 'subject_mismatch',+])++export function hostAccessDenialCodeForReason(reason: unknown): HostAccessDenialCode {+ return typeof reason === 'string' && REVOKED_CONTROL_API_REASONS.has(reason)+ ? 'host_access_revoked'+ : 'host_access_denied'+}++export function hostAccessDenied(code: HostAccessDenialCode): HostAccessDenial {+ return { denied: true, code }+}++/** Narrows `resolveHostConnectionForUser`'s result: true means "respond 403". */+export function isHostAccessDenied<T extends object>(+ resolved: T | HostAccessDenial+): resolved is HostAccessDenial {+ return (resolved as Partial<HostAccessDenial>).denied === true+}++/** The single writer of rpc-proxy's own Host authorization 403. */+export function respondHostAccessDenied(+ res: ExpressResponse,+ code: HostAccessDenialCode | HostAccessDenial+): void {+ res.status(403).json({+ error: HOST_ACCESS_DENIED_ERROR,+ code: typeof code === 'string' ? code : code.code,+ })+}diff --git a/rpc-proxy/src/services/mcpProxyService.ts b/rpc-proxy/src/services/mcpProxyService.tsindex 43551cffb..7cf8595b1 100644--- a/rpc-proxy/src/services/mcpProxyService.ts+++ b/rpc-proxy/src/services/mcpProxyService.ts@@ -6,6 +6,7 @@ import {fetchHostConnectionFromControlApi,fetchUserAllowedServersFromControlApi,} from './controlApiRestService.js'+import { type HostAccessDenial, isHostAccessDenied } from './hostAccessDenial.js'export { validateRpcRequest, forwardRpcToServer } from './mcpRpcService.js'export {@@ -83,11 +84,11 @@ export async function resolveHostConnectionForUser(requestId?: stringdirectRunBinding?: DirectRunBindingRequest}-): Promise<ResolvedServerConnection | null> {+): Promise<ResolvedServerConnection | HostAccessDenial> {const host = await fetchHostConnectionFromControlApi(userId, hostRef, rpcAccessToken, {directRunBinding: edgeContext?.directRunBinding,})- if (!host) return null+ if (isHostAccessDenied(host)) return hostconst headers: Record<string, string> = {...host.headers,diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.tsindex 2b1cdf73a..abe86a550 100644--- a/rpc-proxy/src/services/wakeAndHold.ts+++ b/rpc-proxy/src/services/wakeAndHold.ts@@ -45,9 +45,14 @@ const ENTRY_TTL_MARGIN_MS = 5_000const MAX_TRACKED_WAKE_COORDINATIONS = 1_000const WAKE_SCOPE = 'host:wake:write' as constconst ADMISSION_RETRY_DELAYS_MS = [250, 500, 1_000]-const MIN_ADMISSION_RETRY_TIMEOUT_MS = 100-/** Leaves room for one final upstream timeout before Desktop's 60s deadline. */-export const MAX_REQUEST_HOLD_MS = 48_000+/**+ * An admission retry with less budget than this cannot outlast the upstream's+ * own admission latency: it would time out AFTER the upstream applied the+ * request, turning an applied-once operation into a 504. Below it the request+ * answers a retryable 503 host_waking instead. Bounded by `upstreamTimeoutMs`+ * so a deliberately short upstream timeout still gets a retry.+ */+const MIN_ADMISSION_RETRY_TIMEOUT_MS = 1_000export type WakeHoldOutcome =/** Host is (or just became) reachable — re-issue the upstream request now. */@@ -613,13 +618,28 @@ export type RespondWithWakeAndHoldOptions = {claims: RpcAccessClaims/** Raw bearer forwarded to the wake plane when the caller is wake-capable. */rpcAccessToken: string- /** Re-issues the original upstream request with a deadline-bounded timeout. */- attemptUpstream: (timeoutMs: number) => Promise<void>+ /**+ * Re-issues the original upstream request with a deadline-bounded timeout.+ * The route's FIRST attempt runs without a client-side timeout for mutating+ * calls; `timeoutMs` only ever applies to these hold retries. `deadlineMs`+ * is the absolute request deadline, for phases after the headers (a body+ * transfer) that `timeoutMs` alone does not bound.+ */+ attemptUpstream: (timeoutMs: number, deadlineMs: number) => Promise<void>/** Writes today's error response (502/504) — the pre-wake behavior. */respondLegacy: (error: unknown) => voidcoordinator?: WakeAndHoldCoordinator/** Absolute deadline captured before a route's initial availability probe. */deadlineMs?: number+ /**+ * Keep re-issuing the request (with backoff) until it is accepted or the+ * deadline expires. Only safe when the upstream dedupes a re-presented+ * request: `/messages` carries a per-request `messageId` that mcp-host's+ * admission sink dedupes. Every other route re-issues at most ONCE after the+ * hold: approve/deny/model/cancel have no idempotency key, so each extra+ * POST risks a duplicate side effect. Defaults to false.+ */+ retryUntilDeadline?: boolean}/**@@ -636,8 +656,7 @@ export async function respondWithWakeAndHold(): Promise<void> {const coordinator = options.coordinator ?? hostWakeCoordinatorconst safeHostRef = sanitizeHostRefForLog(options.hostRef)- const deadlineMs =- options.deadlineMs ?? Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)+ const deadlineMs = options.deadlineMs ?? Date.now() + config.wakeMaxHoldMsif (options.res.headersSent) {// A response is already committed for this request: parking it could only// ever produce a duplicate upstream delivery. Refuse loudly.@@ -662,6 +681,9 @@ export async function respondWithWakeAndHold(switch (outcome.kind) {case 'proceed': {let availabilityFailures = 0+ let attempts = 0+ const maxAttempts = options.retryUntilDeadline ? Infinity : 1+ const minAttemptTimeoutMs = Math.min(MIN_ADMISSION_RETRY_TIMEOUT_MS, config.upstreamTimeoutMs)while (Date.now() < deadlineMs) {if (options.res.headersSent) {// Resolution latch: the response was committed while this retry@@ -674,8 +696,9 @@ export async function respondWithWakeAndHold(}try {const remainingMs = deadlineMs - Date.now()- if (remainingMs < MIN_ADMISSION_RETRY_TIMEOUT_MS) break- await options.attemptUpstream(Math.min(config.upstreamTimeoutMs, remainingMs))+ if (remainingMs < minAttemptTimeoutMs) break+ attempts+++ await options.attemptUpstream(Math.min(config.upstreamTimeoutMs, remainingMs), deadlineMs)return} catch (error) {if (options.res.headersSent) {@@ -690,6 +713,9 @@ export async function respondWithWakeAndHold(return}if (isWakeEligibleHostError(error)) {+ // Single-retry routes stop here: the host is still unreachable+ // after the wake, so the caller gets the retryable 503 below.+ if (attempts >= maxAttempts) breakconst remainingMs = deadlineMs - Date.now()const delayMs =ADMISSION_RETRY_DELAYS_MS[@@ -716,7 +742,7 @@ export async function respondWithWakeAndHold(return}console.warn(- `[RPC_PROXY] wake-hold admission deadline exceeded host=${safeHostRef} lastKnownState=${outcome.lastKnownState}`+ `[RPC_PROXY] wake-hold admission ended without an accepted attempt host=${safeHostRef} attempts=${attempts} lastKnownState=${outcome.lastKnownState}`)respondHostWaking(options.res, {hostRef: options.hostRef,diff --git a/rpc-proxy/test/routes.rpc.test.ts b/rpc-proxy/test/routes.rpc.test.tsindex 97352b4f4..73b4e6180 100644--- a/rpc-proxy/test/routes.rpc.test.ts+++ b/rpc-proxy/test/routes.rpc.test.ts@@ -21,7 +21,10 @@ const serviceMock = vi.hoisted(() => ({const controlApiMock = vi.hoisted(() => ({ControlApiHostAccessRejectedError: class ControlApiHostAccessRejectedError extends Error {- constructor(public readonly status: number) {+ constructor(+ public readonly status: number,+ public readonly denialCode: string | null = null+ ) {super(`Control API rejected host access (${status})`)this.name = 'ControlApiHostAccessRejectedError'}@@ -305,12 +308,21 @@ describe('routes/rpc', () => {})it.each([- [401, { error: 'Unauthorized' }],- [403, { error: 'Forbidden: user cannot access this host' }],- [409, { error: 'Direct run attribution conflict' }],+ [401, null, { error: 'Unauthorized' }],+ [+ 403,+ 'host_access_revoked',+ { error: 'Forbidden: user cannot access this host', code: 'host_access_revoked' },+ ],+ [+ 403,+ 'host_access_denied',+ { error: 'Forbidden: user cannot access this host', code: 'host_access_denied' },+ ],+ [409, null, { error: 'Direct run attribution conflict' }],] as const)(- 'preserves a Control API %s from atomic host resolution and never forwards',- async (status, expectedBody) => {+ 'preserves a Control API %s (%s) from atomic host resolution and never forwards',+ async (status, denialCode, expectedBody) => {authTokenMock.verifyRpcToken.mockReturnValue({sub: 'user-1',typ: 'user',@@ -323,7 +335,7 @@ describe('routes/rpc', () => {exp: 9999999999,})serviceMock.resolveHostConnectionForUser.mockRejectedValue(- new controlApiMock.ControlApiHostAccessRejectedError(status)+ new controlApiMock.ControlApiHostAccessRejectedError(status, denialCode))await request(makeApp())@@ -662,7 +674,10 @@ describe('routes/rpc', () => {iat: 1,exp: 9999999999,})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app).get('/rpc/hosts/not-allowed/status')@@ -682,7 +697,10 @@ describe('routes/rpc', () => {iat: 1,exp: 9999999999,})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app).get('/rpc/hosts/not-allowed/health')@@ -702,7 +720,10 @@ describe('routes/rpc', () => {iat: 1,exp: 9999999999,})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app).post('/rpc/hosts/not-allowed/messages')diff --git a/rpc-proxy/test/routes.rpcHostActivityStream.test.ts b/rpc-proxy/test/routes.rpcHostActivityStream.test.tsindex 347b38f15..5ef118f24 100644--- a/rpc-proxy/test/routes.rpcHostActivityStream.test.ts+++ b/rpc-proxy/test/routes.rpcHostActivityStream.test.ts@@ -76,7 +76,10 @@ describe('routes/rpcHostActivityStream', () => {iat: 1,exp: 9999999999,})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app).get('/rpc/hosts/nope/activity/stream')diff --git a/rpc-proxy/test/routes.rpcHostStatusStream.test.ts b/rpc-proxy/test/routes.rpcHostStatusStream.test.tsindex ee0daf362..00d7c88da 100644--- a/rpc-proxy/test/routes.rpcHostStatusStream.test.ts+++ b/rpc-proxy/test/routes.rpcHostStatusStream.test.ts@@ -92,7 +92,10 @@ describe('routes/rpcHostStatusStream', () => {iat: 1,exp: 9999999999,})- serviceMock.resolveHostConnectionForUser.mockResolvedValue(null)+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ denied: true,+ code: 'host_access_denied',+ })const app = makeApp()await request(app).get('/rpc/hosts/nope/status/stream')diff --git a/rpc-proxy/test/services.controlApiHostAccess.test.ts b/rpc-proxy/test/services.controlApiHostAccess.test.tsindex 26e612a56..f6715a209 100644--- a/rpc-proxy/test/services.controlApiHostAccess.test.ts+++ b/rpc-proxy/test/services.controlApiHostAccess.test.ts@@ -61,4 +61,144 @@ describe('control-api canonical host access client', () => {await expect(rejection).rejects.toBeInstanceOf(ControlApiHostAccessRejectedError)await expect(rejection).rejects.toMatchObject({ status })})++ const jsonResponse = (status: number, body: unknown, headers: Record<string, string> = {}) =>+ vi.fn(+ async () =>+ new Response(JSON.stringify(body), {+ status,+ headers: { 'content-type': 'application/json', ...headers },+ })+ )++ // control-api reports the denial reason in this header; its 403 body is always+ // exactly {"error":"Forbidden"}. Pinned literally so the test fails if the two+ // services ever disagree on the name.+ const REASON_HEADER = 'x-host-access-denial-reason'+ const FORBIDDEN_BODY = { error: 'Forbidden' }++ // R1-M6: the denial reason (read from the response header, never the body)+ // decides whether Desktop may treat the loss as a revocation. Only the three+ // reasons that prove removed access map to `host_access_revoked`.+ it.each([+ ['team_membership_missing', 'host_access_revoked'],+ ['directory_grant_missing', 'host_access_revoked'],+ ['subject_mismatch', 'host_access_revoked'],+ ['host_disabled', 'host_access_denied'],+ ['host_missing', 'host_access_denied'],+ ['host_claim_missing', 'host_access_denied'],+ ['unheard_of_reason', 'host_access_denied'],+ ])('a non-direct-run 403 with reason %s resolves to the denial %s', async (reason, code) => {+ const fetchImpl = jsonResponse(403, FORBIDDEN_BODY, { [REASON_HEADER]: reason })++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code })+ expect(fetchImpl).toHaveBeenCalledOnce()+ })++ it('a non-direct-run 403 without the reason header resolves to host_access_denied', async () => {+ const fetchImpl = jsonResponse(403, FORBIDDEN_BODY)++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_denied' })+ expect(fetchImpl).toHaveBeenCalledOnce()+ })++ it('a non-direct-run 403 with an unknown header value resolves to host_access_denied', async () => {+ const fetchImpl = jsonResponse(403, FORBIDDEN_BODY, { [REASON_HEADER]: 'a_reason_added_later' })++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_denied' })+ expect(fetchImpl).toHaveBeenCalledOnce()+ })++ it('a non-direct-run 403 with a non-JSON body still honours the header', async () => {+ const fetchImpl = vi.fn(+ async () =>+ new Response('not json', { status: 403, headers: { [REASON_HEADER]: 'subject_mismatch' } })+ )++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_revoked' })+ })++ it('a reason present only in the body (legacy shape) is ignored', async () => {+ // Proves the body is never read: the same reason maps to host_access_revoked+ // when it arrives as the header (see the table above) but not from the body.+ const fetchImpl = jsonResponse(403, { error: 'Forbidden', reason: 'team_membership_missing' })++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_denied' })+ expect(fetchImpl).toHaveBeenCalledOnce()+ })++ it('a non-direct-run 404 resolves to host_access_denied', async () => {+ const fetchImpl = jsonResponse(404, { error: 'not found' })++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_denied' })+ expect(fetchImpl).toHaveBeenCalledOnce()+ })++ it.each([+ ['userId mismatch', { userId: 'someone-else', hostRef: 'host-a', url: 'http://h:8080' }],+ ['hostRef mismatch', { userId: 'user-1', hostRef: 'host-b', url: 'http://h:8080' }],+ ['empty url', { userId: 'user-1', hostRef: 'host-a', url: ' ' }],+ ])('a 200 with %s resolves to host_access_denied', async (_label, body) => {+ const fetchImpl = jsonResponse(200, body)++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).resolves.toEqual({ denied: true, code: 'host_access_denied' })+ })++ it.each([+ ['team_membership_missing', 'host_access_revoked'],+ ['host_disabled', 'host_access_denied'],+ [undefined, 'host_access_denied'],+ ])('a direct-run 403 (reason %s) throws with denialCode %s', async (reason, code) => {+ const fetchImpl = jsonResponse(403, FORBIDDEN_BODY, reason ? { [REASON_HEADER]: reason } : {})++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ directRunBinding: BINDING,+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).rejects.toMatchObject({ status: 403, denialCode: code })+ })++ it.each([401, 409] as const)('a %s carries no denial code', async status => {+ const fetchImpl = jsonResponse(+ status,+ { error: 'x', reason: 'team_membership_missing' },+ { [REASON_HEADER]: 'team_membership_missing' }+ )++ await expect(+ fetchHostConnectionFromControlApi('user-1', 'host-a', 'signed-rpc-token', {+ directRunBinding: BINDING,+ fetchImpl: fetchImpl as typeof fetch,+ })+ ).rejects.toMatchObject({ status, denialCode: null })+ })})diff --git a/rpc-proxy/test/services.mcpProxyService.test.ts b/rpc-proxy/test/services.mcpProxyService.test.tsindex daf1d2164..678fada2b 100644--- a/rpc-proxy/test/services.mcpProxyService.test.ts+++ b/rpc-proxy/test/services.mcpProxyService.test.ts@@ -242,12 +242,12 @@ describe('services/mcpProxyService', () => {})})- it('returns null on forbidden host resolution', async () => {+ it('resolves a forbidden host to a non-revoking denial when control-api gives no reason', async () => {vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(mkResponse(403, JSON.stringify({ error: 'Forbidden' })))const host = await resolveHostConnectionForUser('user-1', 'agent2', 'rpc-token')- expect(host).toBeNull()+ expect(host).toEqual({ denied: true, code: 'host_access_denied' })})it('forwards host message and returns REST response body', async () => {