Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(rpc-proxy): keep Host error bodies out of route and wake-hold logs

Alfredo Lopez6fa8cb716328
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 074e309d4..d93fc927c 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.10.0',
externalRestApiVersion: '0.1.96',
- rpcProxyVersion: '0.1.121',
+ rpcProxyVersion: '0.1.122',
desktopVersion: '0.10.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index 4aa1fb94d..6c62fd9af 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.121",
+ "version": "0.1.122",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.121",
+ "version": "0.1.122",
"dependencies": {
"@clerum/jwt-key-policy": "file:../packages/jwt-key-policy",
"cors": "^2.8.5",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index b7a5367b0..7d2999260 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.121",
+ "version": "0.1.122",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
index b96b78ec6..10424eaea 100644
--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
@@ -7,7 +7,11 @@ import request from 'supertest'
import { config } from '../config.js'
import { apiErrorHandler } from '../errorHandler.js'
import { createRpcRouter } from '../routes/rpc.js'
-import { forwardHostMessageToHost } from '../services/mcpHostRestService.js'
+import {
+ forwardHostMessageToHost,
+ forwardHostStatus,
+ forwardTaskResultFromHost,
+} from '../services/mcpHostRestService.js'
const authTokenMock = vi.hoisted(() => ({
verifyRpcToken: vi.fn(),
@@ -674,3 +678,202 @@ describe('task result and cancel host-down paths', () => {
expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
})
})
+
+// A Host error body can carry user content (a prompt echo, a file name, a tool
+// argument). rpc-proxy logs which failure happened, never what the Host said:
+// these tests drive the REAL Host REST forwarders against a local Host whose
+// error body holds a unique marker, then search every console line for it.
+describe('Host error bodies stay out of rpc-proxy logs', () => {
+ const CONSOLE_METHODS = ['debug', 'info', 'log', 'warn', 'error'] as const
+ const MARKER_FRAGMENT_WIDTH = 12
+
+ type ScriptedReply = { status: number; body: Record<string, unknown> }
+
+ function captureConsole() {
+ const spies = CONSOLE_METHODS.map(method =>
+ vi.spyOn(console, method).mockImplementation(() => {})
+ )
+ const render = (args: unknown[]) =>
+ args
+ .map(arg => (typeof arg === 'string' ? arg : (JSON.stringify(arg) ?? String(arg))))
+ .join(' ')
+ return {
+ allLines: () => spies.flatMap(spy => spy.mock.calls.map(render)),
+ warnLines: () => spies[CONSOLE_METHODS.indexOf('warn')].mock.calls.map(render),
+ restore: () => spies.forEach(spy => spy.mockRestore()),
+ }
+ }
+
+ function markerFragmentsIn(lines: string[], marker: string): string[] {
+ const hits: string[] = []
+ for (let start = 0; start + MARKER_FRAGMENT_WIDTH <= marker.length; start++) {
+ const fragment = marker.slice(start, start + MARKER_FRAGMENT_WIDTH)
+ if (lines.some(line => line.includes(fragment))) hits.push(fragment)
+ }
+ return hits
+ }
+
+ // Answers hit N with replies[N-1] (the last reply repeats) and points the
+ // resolved host connection at it.
+ async function withScriptedHost(
+ replies: ScriptedReply[],
+ run: (hostHits: () => number) => Promise<void>
+ ): Promise<void> {
+ let hits = 0
+ const server = createServer((req, res) => {
+ hits += 1
+ const reply = replies[Math.min(hits, replies.length) - 1]
+ req.resume()
+ req.on('end', () => {
+ res.writeHead(reply.status, { 'content-type': 'application/json' })
+ res.end(JSON.stringify(reply.body))
+ })
+ })
+ await new Promise<void>((resolve, reject) => {
+ server.once('error', reject)
+ server.listen(0, '127.0.0.1', resolve)
+ })
+ const { port } = server.address() as AddressInfo
+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({
+ ...HOST_CONNECTION,
+ url: `http://127.0.0.1:${port}`,
+ })
+ try {
+ await run(() => hits)
+ } finally {
+ server.closeAllConnections()
+ await new Promise<void>(resolve => server.close(() => resolve()))
+ }
+ }
+
+ function userContentMarker(): string {
+ return `user-content-${randomUUID()}`
+ }
+
+ it('logs the status of a first-attempt Host failure without the Host body', async () => {
+ const marker = userContentMarker()
+ serviceMock.forwardHostMessageToHost.mockImplementation(
+ forwardHostMessageToHost as typeof serviceMock.forwardHostMessageToHost
+ )
+ const logs = captureConsole()
+ try {
+ await withScriptedHost(
+ [{ status: 500, body: { error: 'boom', detail: marker } }],
+ async hostHits => {
+ const response = await postMessage(makeApp())
+
+ expect(hostHits()).toBe(1)
+ expect(response.status).toBe(502)
+ }
+ )
+ const failureLines = logs
+ .warnLines()
+ .filter(line => line.includes('[RPC_PROXY] host message forward failed host=chatllm'))
+ // Witness: the failure path ran and logged exactly once.
+ expect(failureLines).toHaveLength(1)
+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])
+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('logs the status of a post-wake retry Host failure without the Host body', async () => {
+ const marker = userContentMarker()
+ serviceMock.forwardHostMessageToHost.mockImplementation(
+ forwardHostMessageToHost as typeof serviceMock.forwardHostMessageToHost
+ )
+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({
+ kind: 'active',
+ wakeGeneration: 22,
+ })
+ const logs = captureConsole()
+ try {
+ await withScriptedHost(
+ [
+ { status: 503, body: { code: 'host_draining', retryAfterMs: 1000 } },
+ { status: 500, body: { error: 'boom', detail: marker } },
+ ],
+ async hostHits => {
+ const response = await postMessage(makeApp())
+
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(hostHits()).toBe(2)
+ expect(response.status).toBe(502)
+ }
+ )
+ const retryLines = logs
+ .warnLines()
+ .filter(line => line.includes('[RPC_PROXY] wake-hold upstream retry failed host=chatllm'))
+ // Witness: the post-wake retry failure path ran and logged exactly once.
+ expect(retryLines).toHaveLength(1)
+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])
+ expect(retryLines[0]).toContain('error=UpstreamHostError status=500')
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('logs the status of a task result Host failure without the Host body', async () => {
+ const marker = userContentMarker()
+ serviceMock.forwardTaskResultFromHost.mockImplementation(
+ forwardTaskResultFromHost as typeof serviceMock.forwardTaskResultFromHost
+ )
+ const logs = captureConsole()
+ try {
+ await withScriptedHost(
+ [{ status: 500, body: { error: 'boom', detail: marker } }],
+ async hostHits => {
+ const response = await request(makeApp())
+ .get('/rpc/hosts/chatllm/tasks/t-9/result')
+ .set('authorization', 'Bearer token')
+
+ expect(hostHits()).toBe(1)
+ expect(response.status).toBe(502)
+ }
+ )
+ const failureLines = logs
+ .warnLines()
+ .filter(line => line.includes('[RPC_PROXY] task result forward failed'))
+ // Witness: the failure path ran and logged exactly once.
+ expect(failureLines).toHaveLength(1)
+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])
+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('logs the status of a host status read failure without the Host body', async () => {
+ const marker = userContentMarker()
+ authTokenMock.verifyRpcToken.mockReturnValue({
+ ...VALID_CLAIMS,
+ scopes: ['host:status:read'],
+ })
+ serviceMock.forwardHostStatus.mockImplementation(
+ forwardHostStatus as typeof serviceMock.forwardHostStatus
+ )
+ const logs = captureConsole()
+ try {
+ await withScriptedHost(
+ [{ status: 500, body: { error: 'boom', detail: marker } }],
+ async hostHits => {
+ await request(makeApp())
+ .get('/rpc/hosts/chatllm/status')
+ .set('authorization', 'Bearer token')
+
+ expect(hostHits()).toBe(1)
+ }
+ )
+ const failureLines = logs
+ .warnLines()
+ .filter(line => line.includes('[RPC_PROXY] host status failed host=chatllm'))
+ // Witness: the failure path ran and logged exactly once.
+ expect(failureLines).toHaveLength(1)
+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])
+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')
+ } finally {
+ logs.restore()
+ }
+ })
+})
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index cac505d38..ebc4389a1 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -39,6 +39,7 @@ import {
} from '../services/mcpProxyService.js'
import { readMutatingResponseBody } from '../services/upstreamBody.js'
import {
+ describeErrorForLog,
isHostPayloadTooLargeError,
isUpstreamTimeoutError,
isWakeEligibleHostError,
@@ -86,9 +87,9 @@ function parseUnsignedIntegerQuery(value: unknown): number | undefined | null {
export function respondUpstreamUnavailable(res: ExpressResponse, error: unknown): void {
if (res.headersSent) {
console.warn(
- `[RPC_PROXY] suppressing duplicate terminal response (upstream-unavailable): ${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] suppressing duplicate terminal response (upstream-unavailable): ${describeErrorForLog(
+ error
+ )}`
)
return
}
@@ -122,11 +123,7 @@ function respondHostMessageForwardFailure(res: ExpressResponse, error: unknown):
*/
function guardedNext(res: ExpressResponse, next: NextFunction, error: unknown): void {
if (res.headersSent) {
- console.warn(
- `[RPC_PROXY] suppressing post-response route error: ${
- error instanceof Error ? error.message : String(error)
- }`
- )
+ console.warn(`[RPC_PROXY] suppressing post-response route error: ${describeErrorForLog(error)}`)
return
}
next(error)
@@ -526,9 +523,9 @@ export function createRpcRouter(): Router {
})
} catch (error) {
console.warn(
- `[RPC_PROXY] host message forward failed host=${hostRef} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] host message forward failed host=${hostRef} error=${describeErrorForLog(
+ error
+ )}`
)
if (isWakeEligibleHostError(error)) {
// Stateless wake-and-hold: a down or draining host triggers a
@@ -570,9 +567,9 @@ export function createRpcRouter(): Router {
return
}
console.warn(
- `[RPC_PROXY] host message forward failed host=${String(req.params.hostRef || '').trim()} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] host message forward failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(
+ error
+ )}`
)
respondUpstreamUnavailable(res, error)
}
@@ -1354,9 +1351,7 @@ export function createRpcRouter(): Router {
try {
await attemptTaskResult()
} catch (error) {
- console.warn(
- `[RPC_PROXY] task result forward failed error=${error instanceof Error ? error.message : String(error)}`
- )
+ console.warn(`[RPC_PROXY] task result forward failed error=${describeErrorForLog(error)}`)
if (isWakeEligibleHostError(error)) {
await respondWithWakeAndHold({
res,
@@ -1375,9 +1370,7 @@ export function createRpcRouter(): Router {
respondUpstreamUnavailable(res, error)
}
} catch (error) {
- console.warn(
- `[RPC_PROXY] task result forward failed error=${error instanceof Error ? error.message : String(error)}`
- )
+ console.warn(`[RPC_PROXY] task result forward failed error=${describeErrorForLog(error)}`)
respondUpstreamUnavailable(res, error)
}
}
@@ -1436,9 +1429,7 @@ export function createRpcRouter(): Router {
try {
await attemptCancel()
} catch (error) {
- console.warn(
- `[RPC_PROXY] cancel forward failed error=${error instanceof Error ? error.message : String(error)}`
- )
+ console.warn(`[RPC_PROXY] cancel forward failed error=${describeErrorForLog(error)}`)
if (isWakeEligibleHostError(error)) {
await respondWithWakeAndHold({
res,
@@ -1455,9 +1446,7 @@ export function createRpcRouter(): Router {
respondUpstreamUnavailable(res, error)
}
} catch (error) {
- console.warn(
- `[RPC_PROXY] cancel forward failed error=${error instanceof Error ? error.message : String(error)}`
- )
+ console.warn(`[RPC_PROXY] cancel forward failed error=${describeErrorForLog(error)}`)
respondUpstreamUnavailable(res, error)
}
}
@@ -1678,9 +1667,9 @@ export function createRpcRouter(): Router {
res.status(200).json(activity)
} catch (error) {
console.warn(
- `[RPC_PROXY] host activity failed host=${String(req.params.hostRef || '').trim()} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] host activity failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(
+ error
+ )}`
)
next(error)
}
@@ -1719,9 +1708,9 @@ export function createRpcRouter(): Router {
res.status(200).json(status)
} catch (error) {
console.warn(
- `[RPC_PROXY] host status failed host=${String(req.params.hostRef || '').trim()} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] host status failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(
+ error
+ )}`
)
next(error)
}
@@ -1755,9 +1744,9 @@ export function createRpcRouter(): Router {
res.status(200).json(health)
} catch (error) {
console.warn(
- `[RPC_PROXY] host health failed host=${String(req.params.hostRef || '').trim()} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] host health failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(
+ error
+ )}`
)
next(error)
}
diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
index 6739003fd..9c6e355ed 100644
--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
@@ -3,10 +3,12 @@ import type { Response as ExpressResponse } from 'express'
import { config } from '../../config.js'
import type { ResolvedServerConnection, RpcAccessClaims, RpcScope } from '../../types.js'
import type { HostWakeApiResponse } from '../controlApiRestService.js'
+import { UpstreamHostError } from '../mcpHostRestService.js'
import { UpstreamBodyReadError } from '../upstreamBody.js'
import {
WakeAndHoldCoordinator,
type WakeCoordinatorDeps,
+ describeErrorForLog,
isHostDownNetworkError,
isHostDrainingError,
respondWithWakeAndHold,
@@ -443,6 +445,48 @@ describe('host error classification', () => {
})
})
+describe('describeErrorForLog', () => {
+ const HOST_BODY = '{"error":"boom","detail":"user-content-describe-marker"}'
+
+ it('names an UpstreamHostError by status and never by its message', () => {
+ const error = new UpstreamHostError(500, HOST_BODY)
+ // Witness: the message really carries the Host body this helper must drop.
+ expect(error.message).toContain('user-content-describe-marker')
+
+ const described = describeErrorForLog(error)
+
+ expect(described).toBe('UpstreamHostError status=500')
+ expect(described).not.toContain('user-content')
+ })
+
+ it('names a network failure by the code on its cause', () => {
+ const error = new TypeError('fetch failed', { cause: { code: 'ECONNREFUSED' } })
+
+ expect(describeErrorForLog(error)).toBe('TypeError code=ECONNREFUSED')
+ })
+
+ it('names a body failure after the headers by the code of the read it wraps', () => {
+ const read = Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })
+
+ expect(describeErrorForLog(new UpstreamBodyReadError(read))).toBe(
+ 'UpstreamBodyReadError code=ECONNRESET'
+ )
+ })
+
+ it('keeps only identifier characters of a name and code', () => {
+ const error = Object.assign(new Error('x'), {
+ name: 'Bad\nName {"body"}',
+ code: 'E_CODE\r\n<script>',
+ })
+
+ expect(describeErrorForLog(error)).toBe('BadNamebody code=E_CODEscript')
+ })
+
+ it('reports a thrown non-Error by its type only', () => {
+ expect(describeErrorForLog('raw upstream text user-content')).toBe('non-error type=string')
+ })
+})
+
describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () => {
type FakeRes = {
headersSent: boolean
@@ -854,6 +898,30 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
+ it('logs a Host failure raised after the response was committed by status, without its body', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
+ const marker = 'user-content-post-response-marker'
+
+ const res = makeRes()
+ const attemptUpstream = vi.fn(async () => {
+ res.status(200).json({ success: true, taskId: 't-committed' })
+ throw new UpstreamHostError(500, `{"error":"boom","detail":"${marker}"}`)
+ })
+ const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
+
+ await vi.advanceTimersByTimeAsync(5_000)
+ await pending
+ const lines = [...warnSpy.mock.calls, ...debugSpy.mock.calls].map(args => args.join(' '))
+ const suppressed = lines.filter(line =>
+ line.includes('wake-hold post-response failure suppressed (already resolved) host=chatllm')
+ )
+ // Witness: the post-response branch ran and logged exactly once.
+ expect(suppressed).toHaveLength(1)
+ expect(lines.filter(line => line.includes(marker))).toEqual([])
+ expect(suppressed[0]).toContain('error=UpstreamHostError status=500')
+ })
+
it('refuses to park a request whose response is already committed, loudly and without waking', async () => {
const { coordinator, requestWake } = makeCoordinator()
const res = makeRes()
diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.ts
index a2209eda2..a8f3c07b9 100644
--- a/rpc-proxy/src/services/wakeAndHold.ts
+++ b/rpc-proxy/src/services/wakeAndHold.ts
@@ -9,6 +9,35 @@ function sanitizeHostRefForLog(hostRef: string): string {
return hostRef.replace(/[\r\n\t\x00-\x1f\x7f]/g, '')
}
+/** Keeps an error name or code to the characters such identifiers use. */
+function logToken(value: string): string {
+ return value.replace(/[^A-Za-z0-9_.:-]/g, '').slice(0, 64)
+}
+
+/**
+ * Identifies a failed upstream call for a log line WITHOUT its message. An
+ * UpstreamHostError message embeds up to 300 characters of the Host response
+ * body, and a Host error body can carry user content, so no Host-facing log
+ * line may print `error.message`. What tells failures apart survives: the
+ * error name, the upstream HTTP status and the network error code, e.g.
+ * `UpstreamHostError status=500` or `TypeError code=ECONNREFUSED`.
+ */
+export function describeErrorForLog(error: unknown): string {
+ if (!(error instanceof Error)) return `non-error type=${typeof error}`
+ const { status, code, cause } = error as Error & {
+ status?: unknown
+ code?: unknown
+ cause?: unknown
+ }
+ const parts = [logToken(error.name)]
+ if (typeof status === 'number' && Number.isInteger(status)) parts.push(`status=${status}`)
+ const causeCode =
+ cause !== null && typeof cause === 'object' ? (cause as { code?: unknown }).code : undefined
+ const errorCode = typeof code === 'string' ? code : causeCode
+ if (typeof errorCode === 'string') parts.push(`code=${logToken(errorCode)}`)
+ return parts.join(' ')
+}
+
/**
* Stateless wake-and-hold (Stage 5, Issue #791 §11).
*
@@ -744,9 +773,9 @@ export async function respondWithWakeAndHold(
// success write threw): the request is resolved. A retry here is
// the duplicate-delivery bug — never re-forward, fail loudly.
console.warn(
- `[RPC_PROXY] wake-hold post-response failure suppressed (already resolved) host=${safeHostRef} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] wake-hold post-response failure suppressed (already resolved) host=${safeHostRef} error=${describeErrorForLog(
+ error
+ )}`
)
return
}
@@ -765,9 +794,9 @@ export async function respondWithWakeAndHold(
// The host answered with a non-availability failure — exactly
// today's behavior for an up-but-erroring host.
console.warn(
- `[RPC_PROXY] wake-hold upstream retry failed host=${safeHostRef} error=${
- error instanceof Error ? error.message : String(error)
- }`
+ `[RPC_PROXY] wake-hold upstream retry failed host=${safeHostRef} error=${describeErrorForLog(
+ error
+ )}`
)
options.respondLegacy(error)
return