Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(review): address proxy probe, desktop privacy, HCC timer, and workflow recovery findings

Alfredo Lopezb86f873e029f
diff --git a/desktop-app/src/__tests__/chatStore.test.ts b/desktop-app/src/__tests__/chatStore.test.ts
index 1674a5866..80ed44ef4 100644
--- a/desktop-app/src/__tests__/chatStore.test.ts
+++ b/desktop-app/src/__tests__/chatStore.test.ts
@@ -342,6 +342,24 @@ describe('messages', () => {
expect(messages[1]!.content).toBe('hello')
})
+ it('does not load transcript messages while deleted-chat cleanup is pending', async () => {
+ await store.createChat('agent-1', 'pending-delete')
+ await store.saveMessages('agent-1', 'pending-delete', [
+ { id: 'private-message', role: 'user', content: 'still on disk', timestamp: 1000 },
+ ])
+ vi.spyOn(fs, 'rm').mockRejectedValue(transientFileReadError('EACCES'))
+
+ await expect(store.deleteChat('agent-1', 'pending-delete', TEAM_A_SCOPE)).resolves.toEqual({
+ cleanupPending: true,
+ })
+ expect((await store.getIndex('agent-1')).pendingChatCleanup).toContainEqual({
+ chatId: 'pending-delete',
+ authorityScope: TEAM_A_SCOPE,
+ })
+ await expect(store.loadMessages('agent-1', 'pending-delete')).resolves.toEqual([])
+ await expect(fs.access(chatMetaPath('pending-delete'))).resolves.toBeUndefined()
+ })
+
it('upserts an optimistic outgoing message and later updates it by the same id', async () => {
await store.createChat('agent-1', 'upsert-send')
const optimistic = { id: 'outgoing-1', role: 'user' as const, content: 'hello', timestamp: 1 }
diff --git a/desktop-app/src/chatStore.ts b/desktop-app/src/chatStore.ts
index f12e9e1b6..2bba0fd82 100644
--- a/desktop-app/src/chatStore.ts
+++ b/desktop-app/src/chatStore.ts
@@ -1802,6 +1802,8 @@ export class ChatStore {
): Promise<ChatMessage[]> {
try {
return await this.serializeChat(agentRef, chatId, async () => {
+ const index = await this.getIndex(agentRef)
+ if (this.isDeletedInScope(index, chatId)) return []
const meta = await this.readOrMigratePagedChatUnlocked(agentRef, chatId)
if (!meta) return []
return this.readMessagesFromPagedMeta(agentRef, chatId, meta, limit, offset)
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
index 0c41cbcce..a275f386c 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
@@ -210,6 +210,35 @@ describe('confirmed deletion catalog protection', () => {
})
describe('revoked host catalog protection', () => {
+ it('filters held-host sessions from the sidebar while retaining them for recovery', async () => {
+ const blocked = new Set<string>()
+ clerum.chat.getIndex.mockResolvedValue(localIndex([]))
+ clerum.rpc.listSessions.mockResolvedValue(
+ serverSessions([{ agent: 'agent-a', chatId: 'cached-session', title: 'Private title' }])
+ )
+ const controller = renderController({
+ selectedAgent: null,
+ agentNames: ['agent-a'],
+ isHostAccessBlocked: agentRef => blocked.has(agentRef),
+ })
+
+ await waitFor(() =>
+ expect(controller.result.current.latestChatSessions.map(session => session.id)).toEqual([
+ 'cached-session',
+ ])
+ )
+
+ blocked.add('agent-a')
+ controller.rerender()
+ expect(controller.result.current.latestChatSessions).toEqual([])
+
+ blocked.delete('agent-a')
+ controller.rerender()
+ expect(controller.result.current.latestChatSessions.map(session => session.id)).toEqual([
+ 'cached-session',
+ ])
+ })
+
it('hides a nonselected host after 403 and blocks direct reselection', async () => {
const blocked = new Set<string>()
clerum.chat.getIndex.mockResolvedValue(localIndex([]))
diff --git a/desktop-app/ui/src/hooks/domain/useChatListController.ts b/desktop-app/ui/src/hooks/domain/useChatListController.ts
index 1c531ee8d..91c0f94a5 100644
--- a/desktop-app/ui/src/hooks/domain/useChatListController.ts
+++ b/desktop-app/ui/src/hooks/domain/useChatListController.ts
@@ -1567,11 +1567,13 @@ export function useChatListController({
return {
// State (public contract, re-exported unchanged by the parent).
- chatList,
+ chatList: selectedAgent && isHostAccessBlocked(selectedAgent) ? [] : chatList,
chatListLoading,
chatListMoreLoading,
chatListHasMoreRemoteSessions,
- latestChatSessions,
+ latestChatSessions: latestChatSessions.filter(
+ session => !isHostAccessBlocked(session.agentRef)
+ ),
latestChatSessionsLoading,
hideAgent,
// CRUD (public contract).
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index f344dba33..d246d37fa 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.9.0',
externalRestApiVersion: '0.1.84',
- rpcProxyVersion: '0.1.92',
+ rpcProxyVersion: '0.1.93',
desktopVersion: '0.9.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 28f6525e2..0dfe15f5a 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.356",
+ "version": "0.1.357",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.356",
+ "version": "0.1.357",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 0a57cd12e..1acc6e655 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.356",
+ "version": "0.1.357",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 7a28f3350..8591a1f56 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -534,6 +534,8 @@ export class HostReconciler {
private codexSnapshot: CodexCatalogSnapshot = { flagEnabled: false }
private lastCodexConfigMap: k8s.V1ConfigMap | undefined
private readonly readinessTimers: Map<string, ReturnType<typeof setTimeout>> = new Map()
+ private readonly reconcileGenerations = new Map<string, number>()
+ private readonly statusGenerations = new Map<string, number>()
/**
* host.name → image whose pull-policy refusal was already error-logged.
* The periodic resync rebuilds the Deployment forever; the operator needs
@@ -2183,10 +2185,19 @@ export class HostReconciler {
private setStatus(name: string, status: HostRuntimeStatus): void {
this.statusMap.set(name, status)
+ this.statusGenerations.set(name, this.reconcileGenerations.get(name) ?? 0)
+ }
+
+ private advanceReconcileGeneration(name: string): number {
+ const generation = (this.reconcileGenerations.get(name) ?? 0) + 1
+ this.reconcileGenerations.set(name, generation)
+ return generation
}
private clearStatus(name: string): void {
this.statusMap.delete(name)
+ this.reconcileGenerations.delete(name)
+ this.statusGenerations.delete(name)
this.lifecycle.clearHost(name)
const timer = this.readinessTimers.get(name)
if (timer) {
@@ -3477,6 +3488,7 @@ export class HostReconciler {
intervalMs = 5000,
maxAttempts = 12
): void {
+ const pollGeneration = this.reconcileGenerations.get(name) ?? 0
const existing = this.readinessTimers.get(name)
if (existing) {
clearTimeout(existing)
@@ -3489,6 +3501,13 @@ export class HostReconciler {
attempts++
const ready = await this.checkDeploymentReady(name, namespace)
if (ready) {
+ if (
+ (this.reconcileGenerations.get(name) ?? 0) !== pollGeneration ||
+ (this.statusGenerations.get(name) ?? 0) > pollGeneration
+ ) {
+ this.readinessTimers.delete(name)
+ return
+ }
this.setStatus(name, { deployed: true, ready: true, message: 'Running' })
this.readinessTimers.delete(name)
return
@@ -4579,6 +4598,7 @@ export class HostReconciler {
const capturedAuthority = this.captureHostMutationAuthority()
const capturedDependencies = this.resolveHostMutationDependencies?.(host)
return this.lifecycle.serializeByHost(host.name, async () => {
+ this.advanceReconcileGeneration(host.name)
this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)
const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host
if (!admittedHost) return
@@ -4803,6 +4823,12 @@ export class HostReconciler {
: []),
...npFailures,
]
+ this.advanceReconcileGeneration(host.name)
+ const timer = this.readinessTimers.get(host.name)
+ if (timer) {
+ clearTimeout(timer)
+ this.readinessTimers.delete(host.name)
+ }
this.setStatus(host.name, {
deployed: applied,
ready,
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 26b13eb7d..3aa545b8f 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.467",
+ "version": "0.1.468",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.467",
+ "version": "0.1.468",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 23c13306d..dcc62db72 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.467",
+ "version": "0.1.468",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts b/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
index bf58520d0..d31ad928f 100644
--- a/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
+++ b/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
@@ -634,6 +634,35 @@ describe('runToolUseLoop — loop control', () => {
expect(workflowTriggerTool.execute).toHaveBeenCalledTimes(1)
})
+ it('recovers a bare hyphenated workflow slug when the LLM omits workflow_trigger', async () => {
+ const reasoning = createMockReasoning([
+ { type: 'text', content: 'I can help with that.' },
+ {
+ type: 'tool_calls',
+ calls: [{ id: 'tc_1', name: 'workflow_trigger', arguments: { name: 'risk-review' } }],
+ },
+ { type: 'text', content: 'The workflow request is now being handled.' },
+ ])
+ const workflowTriggerTool = createMockTool('workflow_trigger', {
+ sanitize: false,
+ output: JSON.stringify({ workflowName: 'risk-review', phase: 'Pending' }),
+ })
+ const config = buildLoopConfig({
+ reasoning,
+ toolRegistry: createMockRegistry([workflowTriggerTool]),
+ safety: new BasicSafety(),
+ events: new SimpleEventEmitter(),
+ conversation: makeFakeConversation(),
+ maxIterations: 4,
+ })
+
+ const result = await runToolUseLoop(config, [{ role: 'user', content: 'Run risk-review' }])
+
+ expect(result.type).toBe('response')
+ expect(reasoning.respondWithTools).toHaveBeenCalledTimes(2)
+ expect(workflowTriggerTool.execute).toHaveBeenCalledTimes(1)
+ })
+
it('synthesizes a workflow_list answer when the LLM omits returned recipe names', async () => {
const reasoning = createMockReasoning([
{
diff --git a/mcp-host/src/core/orchestration/toolUseLoopIntentRecovery.ts b/mcp-host/src/core/orchestration/toolUseLoopIntentRecovery.ts
index 00bca5031..c5a622354 100644
--- a/mcp-host/src/core/orchestration/toolUseLoopIntentRecovery.ts
+++ b/mcp-host/src/core/orchestration/toolUseLoopIntentRecovery.ts
@@ -89,10 +89,15 @@ export function shouldRecoverWorkflowTriggerTextResponse(
if (isWorkflowListIntent(userText)) return false
const normalizedUser = userText.toLowerCase()
+ const directTrigger = userText.match(/^\s*(run|execute|trigger|start)\s+(.+?)\s*$/i)
+ const directCandidate = directTrigger?.[2].replace(/[\u0060'"“”‘’]/g, '').trim()
+ const namesDirectRecipe =
+ directCandidate !== undefined && /^[a-z0-9]+(?:-[a-z0-9]+)+$/i.test(directCandidate)
const asksToTrigger = /\b(run|start|trigger|execute|launch)\b/.test(normalizedUser)
const includesRecipeLikeName = /\b[a-z0-9]+(?:-[a-z0-9]+)+\b/.test(normalizedUser)
const namesWorkflow =
/\bworkflow\s+recipe\b/.test(normalizedUser) ||
+ namesDirectRecipe ||
(includesRecipeLikeName && /\b(?:workflow|recipe)\b/.test(normalizedUser))
if (!asksToTrigger || !namesWorkflow) return false
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index b4ebe5f81..0ea81ec4f 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.92",
+ "version": "0.1.93",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.92",
+ "version": "0.1.93",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index 325a2ebfd..6c24211c3 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.92",
+ "version": "0.1.93",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
index 0fd4d72af..f7309d4df 100644
--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
@@ -113,7 +113,7 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
const response = await postMessage(makeApp()).expect(200)
expect(response.body).toEqual({ success: true, taskId: 't-1' })
- expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(2)
expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledWith('chatllm', 'token')
expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)
})
@@ -149,9 +149,9 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
.expect(200)
.then(response => response)
await firstForward
- expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
- await vi.advanceTimersByTimeAsync(750)
expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(750)
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(2)
expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(1_500)
upstreamAdmits = true
@@ -268,7 +268,7 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
const response = await postMessage(makeApp()).expect(200)
expect(response.body).toEqual({ success: true, taskId: 't-2' })
- expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(2)
})
it('drain-cancel bounce resolves within the short retry schedule (250ms then 1s)', async () => {
@@ -287,13 +287,15 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(3)
})
- it('409 not-stateless keeps today behavior exactly: 502 Upstream host unavailable', async () => {
+ it('409 not-stateless keeps the full upstream timeout and legacy 502 behavior', async () => {
serviceMock.forwardHostMessageToHost.mockRejectedValue(hostDownError())
controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({ kind: 'not-stateless' })
const response = await postMessage(makeApp()).expect(502)
expect(response.body).toEqual({ error: 'Upstream host unavailable' })
+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][2]).toEqual({ async: false })
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(2)
})
it('404 unknown host maps to a 404 for the caller', async () => {
@@ -348,7 +350,7 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(serviceMock.forwardHostMessageToHost.mock.calls.length).toBeGreaterThan(4)
})
- it('a non-availability upstream failure (mcp-host 500) never triggers a wake', async () => {
+ it('a non-availability upstream failure (mcp-host 500) never enters the wake hold', async () => {
serviceMock.forwardHostMessageToHost.mockRejectedValue(
new serviceMock.UpstreamHostError(500, '{"error":"boom"}')
)
@@ -356,10 +358,10 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
const response = await postMessage(makeApp()).expect(502)
expect(response.body).toEqual({ error: 'Upstream host unavailable' })
- expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1) // statelessness check only
})
- it('an upstream AbortError keeps the 504 path and never triggers a wake', async () => {
+ it('an upstream AbortError keeps the 504 path without entering the wake hold', async () => {
const abort = new Error('aborted')
abort.name = 'AbortError'
serviceMock.forwardHostMessageToHost.mockRejectedValue(abort)
@@ -367,7 +369,7 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
const response = await postMessage(makeApp()).expect(504)
expect(response.body).toEqual({ error: 'Gateway Timeout' })
- expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1) // statelessness check only
})
it('a token without wake scope keeps the original full upstream timeout', async () => {
@@ -385,6 +387,17 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(serviceMock.forwardHostMessageToHost.mock.calls[0][2]).toEqual({ async: false })
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
+
+ it('uses the full upstream timeout when statelessness cannot be confirmed', async () => {
+ controlApiMock.requestHostWakeFromControlApi.mockRejectedValue(
+ new Error('wake API unavailable')
+ )
+ serviceMock.forwardHostMessageToHost.mockResolvedValue({ success: true, taskId: 't-stateful' })
+
+ await postMessage(makeApp()).expect(200)
+
+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][2]).toEqual({ async: false })
+ })
})
describe('task result and cancel host-down paths', () => {
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index 829721aec..8bf69f94c 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -47,7 +47,7 @@ const RFC1123_RE = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/
// own copy (the two services share no package, so each names its own).
const SESSIONS_LIMIT_CAP = 100
const MESSAGES_LIMIT_CAP = 200
-/** A suspended pod may spend seconds failing DNS/connect before the wake starts. */
+/** A confirmed stateless host may be suspended and spend seconds failing DNS/connect. */
const MESSAGE_AVAILABILITY_PROBE_MS = 750
function isSafeUpstreamPathSegment(value: string): boolean {
@@ -443,10 +443,28 @@ export function createRpcRouter(): Router {
)
let upstreamResponse: Record<string, unknown> | null = null
const wakeCapable = isWakeCapable(auth, hostRef, Date.now())
+ let statelessHostConfirmed = false
+ if (wakeCapable) {
+ try {
+ const wakeStatus = await requestHostWakeFromControlApi(hostRef, rpcAccessToken)
+ // The wake API checks lifecycle.spec.stateless before returning either
+ // status. Unknown, rate-limited, unauthorized, and failed checks do not
+ // authorize a short timeout for a mutating message request.
+ statelessHostConfirmed =
+ wakeStatus.kind === 'active' || wakeStatus.kind === 'wake-requested'
+ } catch {
+ // A failed statelessness check must not turn a healthy stateful send
+ // into a short, aborting POST. The normal timeout still allows the
+ // existing wake path to handle a genuine upstream availability error.
+ }
+ }
try {
+ // This is a mutating POST. Its short availability probe is safe only
+ // after the wake API confirms a stateless Host, where deliveryMessageId
+ // lets mcp-host deduplicate an interrupted send and its retry.
upstreamResponse = await forwardHostMessageToHost(host, forwardedBody, {
async: isAsync,
- ...(wakeCapable ? { timeoutMs: MESSAGE_AVAILABILITY_PROBE_MS } : {}),
+ ...(statelessHostConfirmed ? { timeoutMs: MESSAGE_AVAILABILITY_PROBE_MS } : {}),
})
} catch (error) {
console.warn(
@@ -470,9 +488,10 @@ export function createRpcRouter(): Router {
claims: auth,
rpcAccessToken,
deadlineMs: wakeDeadlineMs,
- attemptUpstream: async () => {
+ attemptUpstream: async timeoutMs => {
const retried = await forwardHostMessageToHost(host, forwardedBody, {
async: isAsync,
+ timeoutMs,
})
res.status(200).json(retried)
},
diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
index da48afd64..85fe77723 100644
--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { Response as ExpressResponse } from 'express'
+import { config } from '../../config.js'
import type { ResolvedServerConnection, RpcAccessClaims, RpcScope } from '../../types.js'
import type { HostWakeApiResponse } from '../controlApiRestService.js'
import {
@@ -474,7 +475,7 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
function respondOptions(
coordinator: WakeAndHoldCoordinator,
res: FakeRes,
- attemptUpstream: () => Promise<void>,
+ attemptUpstream: (timeoutMs: number) => Promise<void>,
overrides?: {
hostRef?: string
host?: ResolvedServerConnection
@@ -626,6 +627,43 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
expect(attemptUpstream).toHaveBeenCalledTimes(3)
})
+ it('passes each admission retry only the remaining hold budget', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
+ const res = makeRes()
+ const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
+ throw upstreamDrainingError()
+ })
+ const pending = respondWithWakeAndHold({
+ ...respondOptions(coordinator, res, attemptUpstream),
+ deadlineMs: Date.now() + 500,
+ })
+
+ await vi.advanceTimersByTimeAsync(500)
+ await pending
+
+ expect(attemptUpstream.mock.calls.map(([timeoutMs]) => timeoutMs)).toEqual([500, 250])
+ expect(res.statusCode).toBe(503)
+ })
+
+ it('caps an admission retry timeout at the configured upstream timeout', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
+ const res = makeRes()
+ const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
+ res.status(200).json({ success: true, taskId: 't-capped' })
+ })
+ const pending = respondWithWakeAndHold({
+ ...respondOptions(coordinator, res, attemptUpstream),
+ deadlineMs: Date.now() + config.upstreamTimeoutMs + 5_000,
+ })
+
+ await pending
+
+ expect(attemptUpstream).toHaveBeenCalledWith(config.upstreamTimeoutMs)
+ expect(res.statusCode).toBe(200)
+ })
+
it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.ts
index f9f675cfd..d4c885f33 100644
--- a/rpc-proxy/src/services/wakeAndHold.ts
+++ b/rpc-proxy/src/services/wakeAndHold.ts
@@ -604,8 +604,8 @@ export type RespondWithWakeAndHoldOptions = {
claims: RpcAccessClaims
/** Raw bearer forwarded to the wake plane when the caller is wake-capable. */
rpcAccessToken: string
- /** Re-issues the original upstream request and writes the success response. */
- attemptUpstream: () => Promise<void>
+ /** Re-issues the original upstream request with a deadline-bounded timeout. */
+ attemptUpstream: (timeoutMs: number) => Promise<void>
/** Writes today's error response (502/504) — the pre-wake behavior. */
respondLegacy: (error: unknown) => void
coordinator?: WakeAndHoldCoordinator
@@ -664,7 +664,9 @@ export async function respondWithWakeAndHold(
return
}
try {
- await options.attemptUpstream()
+ const remainingMs = deadlineMs - Date.now()
+ if (remainingMs <= 0) break
+ await options.attemptUpstream(Math.min(config.upstreamTimeoutMs, remainingMs))
return
} catch (error) {
if (options.res.headersSent) {
diff --git a/scripts/e2e/e2e-hcc-stateless-cache-containment.sh b/scripts/e2e/e2e-hcc-stateless-cache-containment.sh
index 40dc38717..d850b3236 100755
--- a/scripts/e2e/e2e-hcc-stateless-cache-containment.sh
+++ b/scripts/e2e/e2e-hcc-stateless-cache-containment.sh
@@ -129,6 +129,14 @@ K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-
node -e 'process.exit(require("node:net").isIP(process.argv[1])===4?0:1)' "$K8S_API_SERVICE_HOST" ||
die 'Kubernetes API service address is not IPv4'
K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"
+EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
+RPC_BASE="${RPC_PROXY_BASE_URL%/}"
+DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"
+DEV_PASSWORD="${E2E_USER_PASSWORD:-}"
+if [ -z "$DEV_PASSWORD" ]; then
+ DEV_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
+fi
+[ -n "$DEV_PASSWORD" ] || die 'known-session proof requires E2E_USER_PASSWORD or a canonical admin password'
hcc_pr_a_preflight
create_hcc_api_proxy
HCC_PR_A_SELECTIVE_CHANNEL=1 hcc_pr_a_enable_proxy
@@ -140,14 +148,6 @@ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/n
read -r HCC_UID HCC_RESTARTS <<<"$(wait_for_hcc_identity 30)"
start_hcc_recovery_log_stream
-EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
-RPC_BASE="${RPC_PROXY_BASE_URL%/}"
-DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"
-DEV_PASSWORD="${E2E_USER_PASSWORD:-}"
-if [ -z "$DEV_PASSWORD" ]; then
- DEV_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
-fi
-[ -n "$DEV_PASSWORD" ] || die 'known-session proof requires E2E_USER_PASSWORD or a canonical admin password'
curl -fsS -m 10 "${EXT_BASE}/health" >/dev/null || die 'external-rest-api unavailable'
curl -fsS -m 10 "${RPC_BASE}/health" >/dev/null || die 'rpc-proxy unavailable'
login="$(curl -fsS -m 20 -X POST "${EXT_BASE}/api/v1/auth/password-login" \