Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(rpc-proxy): keep Host error bodies out of route and wake-hold logs
Alfredo Lopez6fa8cb716328
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex 074e309d4..d93fc927c 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.10.0',externalRestApiVersion: '0.1.96',- rpcProxyVersion: '0.1.121',+ rpcProxyVersion: '0.1.122',desktopVersion: '0.10.0',minimumDesktopVersion: '0.1.252',}diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex 4aa1fb94d..6c62fd9af 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.121",+ "version": "0.1.122","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.121",+ "version": "0.1.122","dependencies": {"@clerum/jwt-key-policy": "file:../packages/jwt-key-policy","cors": "^2.8.5",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex b7a5367b0..7d2999260 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.121",+ "version": "0.1.122","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.tsindex b96b78ec6..10424eaea 100644--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts@@ -7,7 +7,11 @@ import request from 'supertest'import { config } from '../config.js'import { apiErrorHandler } from '../errorHandler.js'import { createRpcRouter } from '../routes/rpc.js'-import { forwardHostMessageToHost } from '../services/mcpHostRestService.js'+import {+ forwardHostMessageToHost,+ forwardHostStatus,+ forwardTaskResultFromHost,+} from '../services/mcpHostRestService.js'const authTokenMock = vi.hoisted(() => ({verifyRpcToken: vi.fn(),@@ -674,3 +678,202 @@ describe('task result and cancel host-down paths', () => {expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)})})++// A Host error body can carry user content (a prompt echo, a file name, a tool+// argument). rpc-proxy logs which failure happened, never what the Host said:+// these tests drive the REAL Host REST forwarders against a local Host whose+// error body holds a unique marker, then search every console line for it.+describe('Host error bodies stay out of rpc-proxy logs', () => {+ const CONSOLE_METHODS = ['debug', 'info', 'log', 'warn', 'error'] as const+ const MARKER_FRAGMENT_WIDTH = 12++ type ScriptedReply = { status: number; body: Record<string, unknown> }++ function captureConsole() {+ const spies = CONSOLE_METHODS.map(method =>+ vi.spyOn(console, method).mockImplementation(() => {})+ )+ const render = (args: unknown[]) =>+ args+ .map(arg => (typeof arg === 'string' ? arg : (JSON.stringify(arg) ?? String(arg))))+ .join(' ')+ return {+ allLines: () => spies.flatMap(spy => spy.mock.calls.map(render)),+ warnLines: () => spies[CONSOLE_METHODS.indexOf('warn')].mock.calls.map(render),+ restore: () => spies.forEach(spy => spy.mockRestore()),+ }+ }++ function markerFragmentsIn(lines: string[], marker: string): string[] {+ const hits: string[] = []+ for (let start = 0; start + MARKER_FRAGMENT_WIDTH <= marker.length; start++) {+ const fragment = marker.slice(start, start + MARKER_FRAGMENT_WIDTH)+ if (lines.some(line => line.includes(fragment))) hits.push(fragment)+ }+ return hits+ }++ // Answers hit N with replies[N-1] (the last reply repeats) and points the+ // resolved host connection at it.+ async function withScriptedHost(+ replies: ScriptedReply[],+ run: (hostHits: () => number) => Promise<void>+ ): Promise<void> {+ let hits = 0+ const server = createServer((req, res) => {+ hits += 1+ const reply = replies[Math.min(hits, replies.length) - 1]+ req.resume()+ req.on('end', () => {+ res.writeHead(reply.status, { 'content-type': 'application/json' })+ res.end(JSON.stringify(reply.body))+ })+ })+ await new Promise<void>((resolve, reject) => {+ server.once('error', reject)+ server.listen(0, '127.0.0.1', resolve)+ })+ const { port } = server.address() as AddressInfo+ serviceMock.resolveHostConnectionForUser.mockResolvedValue({+ ...HOST_CONNECTION,+ url: `http://127.0.0.1:${port}`,+ })+ try {+ await run(() => hits)+ } finally {+ server.closeAllConnections()+ await new Promise<void>(resolve => server.close(() => resolve()))+ }+ }++ function userContentMarker(): string {+ return `user-content-${randomUUID()}`+ }++ it('logs the status of a first-attempt Host failure without the Host body', async () => {+ const marker = userContentMarker()+ serviceMock.forwardHostMessageToHost.mockImplementation(+ forwardHostMessageToHost as typeof serviceMock.forwardHostMessageToHost+ )+ const logs = captureConsole()+ try {+ await withScriptedHost(+ [{ status: 500, body: { error: 'boom', detail: marker } }],+ async hostHits => {+ const response = await postMessage(makeApp())++ expect(hostHits()).toBe(1)+ expect(response.status).toBe(502)+ }+ )+ const failureLines = logs+ .warnLines()+ .filter(line => line.includes('[RPC_PROXY] host message forward failed host=chatllm'))+ // Witness: the failure path ran and logged exactly once.+ expect(failureLines).toHaveLength(1)+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')+ } finally {+ logs.restore()+ }+ })++ it('logs the status of a post-wake retry Host failure without the Host body', async () => {+ const marker = userContentMarker()+ serviceMock.forwardHostMessageToHost.mockImplementation(+ forwardHostMessageToHost as typeof serviceMock.forwardHostMessageToHost+ )+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({+ kind: 'active',+ wakeGeneration: 22,+ })+ const logs = captureConsole()+ try {+ await withScriptedHost(+ [+ { status: 503, body: { code: 'host_draining', retryAfterMs: 1000 } },+ { status: 500, body: { error: 'boom', detail: marker } },+ ],+ async hostHits => {+ const response = await postMessage(makeApp())++ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(hostHits()).toBe(2)+ expect(response.status).toBe(502)+ }+ )+ const retryLines = logs+ .warnLines()+ .filter(line => line.includes('[RPC_PROXY] wake-hold upstream retry failed host=chatllm'))+ // Witness: the post-wake retry failure path ran and logged exactly once.+ expect(retryLines).toHaveLength(1)+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])+ expect(retryLines[0]).toContain('error=UpstreamHostError status=500')+ } finally {+ logs.restore()+ }+ })++ it('logs the status of a task result Host failure without the Host body', async () => {+ const marker = userContentMarker()+ serviceMock.forwardTaskResultFromHost.mockImplementation(+ forwardTaskResultFromHost as typeof serviceMock.forwardTaskResultFromHost+ )+ const logs = captureConsole()+ try {+ await withScriptedHost(+ [{ status: 500, body: { error: 'boom', detail: marker } }],+ async hostHits => {+ const response = await request(makeApp())+ .get('/rpc/hosts/chatllm/tasks/t-9/result')+ .set('authorization', 'Bearer token')++ expect(hostHits()).toBe(1)+ expect(response.status).toBe(502)+ }+ )+ const failureLines = logs+ .warnLines()+ .filter(line => line.includes('[RPC_PROXY] task result forward failed'))+ // Witness: the failure path ran and logged exactly once.+ expect(failureLines).toHaveLength(1)+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')+ } finally {+ logs.restore()+ }+ })++ it('logs the status of a host status read failure without the Host body', async () => {+ const marker = userContentMarker()+ authTokenMock.verifyRpcToken.mockReturnValue({+ ...VALID_CLAIMS,+ scopes: ['host:status:read'],+ })+ serviceMock.forwardHostStatus.mockImplementation(+ forwardHostStatus as typeof serviceMock.forwardHostStatus+ )+ const logs = captureConsole()+ try {+ await withScriptedHost(+ [{ status: 500, body: { error: 'boom', detail: marker } }],+ async hostHits => {+ await request(makeApp())+ .get('/rpc/hosts/chatllm/status')+ .set('authorization', 'Bearer token')++ expect(hostHits()).toBe(1)+ }+ )+ const failureLines = logs+ .warnLines()+ .filter(line => line.includes('[RPC_PROXY] host status failed host=chatllm'))+ // Witness: the failure path ran and logged exactly once.+ expect(failureLines).toHaveLength(1)+ expect(markerFragmentsIn(logs.allLines(), marker)).toEqual([])+ expect(failureLines[0]).toContain('error=UpstreamHostError status=500')+ } finally {+ logs.restore()+ }+ })+})diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.tsindex cac505d38..ebc4389a1 100644--- a/rpc-proxy/src/routes/rpc.ts+++ b/rpc-proxy/src/routes/rpc.ts@@ -39,6 +39,7 @@ import {} from '../services/mcpProxyService.js'import { readMutatingResponseBody } from '../services/upstreamBody.js'import {+ describeErrorForLog,isHostPayloadTooLargeError,isUpstreamTimeoutError,isWakeEligibleHostError,@@ -86,9 +87,9 @@ function parseUnsignedIntegerQuery(value: unknown): number | undefined | null {export function respondUpstreamUnavailable(res: ExpressResponse, error: unknown): void {if (res.headersSent) {console.warn(- `[RPC_PROXY] suppressing duplicate terminal response (upstream-unavailable): ${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] suppressing duplicate terminal response (upstream-unavailable): ${describeErrorForLog(+ error+ )}`)return}@@ -122,11 +123,7 @@ function respondHostMessageForwardFailure(res: ExpressResponse, error: unknown):*/function guardedNext(res: ExpressResponse, next: NextFunction, error: unknown): void {if (res.headersSent) {- console.warn(- `[RPC_PROXY] suppressing post-response route error: ${- error instanceof Error ? error.message : String(error)- }`- )+ console.warn(`[RPC_PROXY] suppressing post-response route error: ${describeErrorForLog(error)}`)return}next(error)@@ -526,9 +523,9 @@ export function createRpcRouter(): Router {})} catch (error) {console.warn(- `[RPC_PROXY] host message forward failed host=${hostRef} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] host message forward failed host=${hostRef} error=${describeErrorForLog(+ error+ )}`)if (isWakeEligibleHostError(error)) {// Stateless wake-and-hold: a down or draining host triggers a@@ -570,9 +567,9 @@ export function createRpcRouter(): Router {return}console.warn(- `[RPC_PROXY] host message forward failed host=${String(req.params.hostRef || '').trim()} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] host message forward failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(+ error+ )}`)respondUpstreamUnavailable(res, error)}@@ -1354,9 +1351,7 @@ export function createRpcRouter(): Router {try {await attemptTaskResult()} catch (error) {- console.warn(- `[RPC_PROXY] task result forward failed error=${error instanceof Error ? error.message : String(error)}`- )+ console.warn(`[RPC_PROXY] task result forward failed error=${describeErrorForLog(error)}`)if (isWakeEligibleHostError(error)) {await respondWithWakeAndHold({res,@@ -1375,9 +1370,7 @@ export function createRpcRouter(): Router {respondUpstreamUnavailable(res, error)}} catch (error) {- console.warn(- `[RPC_PROXY] task result forward failed error=${error instanceof Error ? error.message : String(error)}`- )+ console.warn(`[RPC_PROXY] task result forward failed error=${describeErrorForLog(error)}`)respondUpstreamUnavailable(res, error)}}@@ -1436,9 +1429,7 @@ export function createRpcRouter(): Router {try {await attemptCancel()} catch (error) {- console.warn(- `[RPC_PROXY] cancel forward failed error=${error instanceof Error ? error.message : String(error)}`- )+ console.warn(`[RPC_PROXY] cancel forward failed error=${describeErrorForLog(error)}`)if (isWakeEligibleHostError(error)) {await respondWithWakeAndHold({res,@@ -1455,9 +1446,7 @@ export function createRpcRouter(): Router {respondUpstreamUnavailable(res, error)}} catch (error) {- console.warn(- `[RPC_PROXY] cancel forward failed error=${error instanceof Error ? error.message : String(error)}`- )+ console.warn(`[RPC_PROXY] cancel forward failed error=${describeErrorForLog(error)}`)respondUpstreamUnavailable(res, error)}}@@ -1678,9 +1667,9 @@ export function createRpcRouter(): Router {res.status(200).json(activity)} catch (error) {console.warn(- `[RPC_PROXY] host activity failed host=${String(req.params.hostRef || '').trim()} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] host activity failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(+ error+ )}`)next(error)}@@ -1719,9 +1708,9 @@ export function createRpcRouter(): Router {res.status(200).json(status)} catch (error) {console.warn(- `[RPC_PROXY] host status failed host=${String(req.params.hostRef || '').trim()} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] host status failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(+ error+ )}`)next(error)}@@ -1755,9 +1744,9 @@ export function createRpcRouter(): Router {res.status(200).json(health)} catch (error) {console.warn(- `[RPC_PROXY] host health failed host=${String(req.params.hostRef || '').trim()} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] host health failed host=${String(req.params.hostRef || '').trim()} error=${describeErrorForLog(+ error+ )}`)next(error)}diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.tsindex 6739003fd..9c6e355ed 100644--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts@@ -3,10 +3,12 @@ import type { Response as ExpressResponse } from 'express'import { config } from '../../config.js'import type { ResolvedServerConnection, RpcAccessClaims, RpcScope } from '../../types.js'import type { HostWakeApiResponse } from '../controlApiRestService.js'+import { UpstreamHostError } from '../mcpHostRestService.js'import { UpstreamBodyReadError } from '../upstreamBody.js'import {WakeAndHoldCoordinator,type WakeCoordinatorDeps,+ describeErrorForLog,isHostDownNetworkError,isHostDrainingError,respondWithWakeAndHold,@@ -443,6 +445,48 @@ describe('host error classification', () => {})})+describe('describeErrorForLog', () => {+ const HOST_BODY = '{"error":"boom","detail":"user-content-describe-marker"}'++ it('names an UpstreamHostError by status and never by its message', () => {+ const error = new UpstreamHostError(500, HOST_BODY)+ // Witness: the message really carries the Host body this helper must drop.+ expect(error.message).toContain('user-content-describe-marker')++ const described = describeErrorForLog(error)++ expect(described).toBe('UpstreamHostError status=500')+ expect(described).not.toContain('user-content')+ })++ it('names a network failure by the code on its cause', () => {+ const error = new TypeError('fetch failed', { cause: { code: 'ECONNREFUSED' } })++ expect(describeErrorForLog(error)).toBe('TypeError code=ECONNREFUSED')+ })++ it('names a body failure after the headers by the code of the read it wraps', () => {+ const read = Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })++ expect(describeErrorForLog(new UpstreamBodyReadError(read))).toBe(+ 'UpstreamBodyReadError code=ECONNRESET'+ )+ })++ it('keeps only identifier characters of a name and code', () => {+ const error = Object.assign(new Error('x'), {+ name: 'Bad\nName {"body"}',+ code: 'E_CODE\r\n<script>',+ })++ expect(describeErrorForLog(error)).toBe('BadNamebody code=E_CODEscript')+ })++ it('reports a thrown non-Error by its type only', () => {+ expect(describeErrorForLog('raw upstream text user-content')).toBe('non-error type=string')+ })+})+describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () => {type FakeRes = {headersSent: boolean@@ -854,6 +898,30 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>expect(coordinator.trackedCoordinationCount()).toBe(0)})+ it('logs a Host failure raised after the response was committed by status, without its body', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })+ const marker = 'user-content-post-response-marker'++ const res = makeRes()+ const attemptUpstream = vi.fn(async () => {+ res.status(200).json({ success: true, taskId: 't-committed' })+ throw new UpstreamHostError(500, `{"error":"boom","detail":"${marker}"}`)+ })+ const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))++ await vi.advanceTimersByTimeAsync(5_000)+ await pending+ const lines = [...warnSpy.mock.calls, ...debugSpy.mock.calls].map(args => args.join(' '))+ const suppressed = lines.filter(line =>+ line.includes('wake-hold post-response failure suppressed (already resolved) host=chatllm')+ )+ // Witness: the post-response branch ran and logged exactly once.+ expect(suppressed).toHaveLength(1)+ expect(lines.filter(line => line.includes(marker))).toEqual([])+ expect(suppressed[0]).toContain('error=UpstreamHostError status=500')+ })+it('refuses to park a request whose response is already committed, loudly and without waking', async () => {const { coordinator, requestWake } = makeCoordinator()const res = makeRes()diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.tsindex a2209eda2..a8f3c07b9 100644--- a/rpc-proxy/src/services/wakeAndHold.ts+++ b/rpc-proxy/src/services/wakeAndHold.ts@@ -9,6 +9,35 @@ function sanitizeHostRefForLog(hostRef: string): string {return hostRef.replace(/[\r\n\t\x00-\x1f\x7f]/g, '')}+/** Keeps an error name or code to the characters such identifiers use. */+function logToken(value: string): string {+ return value.replace(/[^A-Za-z0-9_.:-]/g, '').slice(0, 64)+}++/**+ * Identifies a failed upstream call for a log line WITHOUT its message. An+ * UpstreamHostError message embeds up to 300 characters of the Host response+ * body, and a Host error body can carry user content, so no Host-facing log+ * line may print `error.message`. What tells failures apart survives: the+ * error name, the upstream HTTP status and the network error code, e.g.+ * `UpstreamHostError status=500` or `TypeError code=ECONNREFUSED`.+ */+export function describeErrorForLog(error: unknown): string {+ if (!(error instanceof Error)) return `non-error type=${typeof error}`+ const { status, code, cause } = error as Error & {+ status?: unknown+ code?: unknown+ cause?: unknown+ }+ const parts = [logToken(error.name)]+ if (typeof status === 'number' && Number.isInteger(status)) parts.push(`status=${status}`)+ const causeCode =+ cause !== null && typeof cause === 'object' ? (cause as { code?: unknown }).code : undefined+ const errorCode = typeof code === 'string' ? code : causeCode+ if (typeof errorCode === 'string') parts.push(`code=${logToken(errorCode)}`)+ return parts.join(' ')+}+/*** Stateless wake-and-hold (Stage 5, Issue #791 §11).*@@ -744,9 +773,9 @@ export async function respondWithWakeAndHold(// success write threw): the request is resolved. A retry here is// the duplicate-delivery bug — never re-forward, fail loudly.console.warn(- `[RPC_PROXY] wake-hold post-response failure suppressed (already resolved) host=${safeHostRef} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] wake-hold post-response failure suppressed (already resolved) host=${safeHostRef} error=${describeErrorForLog(+ error+ )}`)return}@@ -765,9 +794,9 @@ export async function respondWithWakeAndHold(// The host answered with a non-availability failure — exactly// today's behavior for an up-but-erroring host.console.warn(- `[RPC_PROXY] wake-hold upstream retry failed host=${safeHostRef} error=${- error instanceof Error ? error.message : String(error)- }`+ `[RPC_PROXY] wake-hold upstream retry failed host=${safeHostRef} error=${describeErrorForLog(+ error+ )}`)options.respondLegacy(error)return