Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(review): address zach88 and jozer-ami blockers and cheap lows

Alfredo Lopez378d74f7403f
diff --git a/desktop-app/src/__tests__/chatStore.test.ts b/desktop-app/src/__tests__/chatStore.test.ts
index 80ed44ef4..c861d6ae9 100644
--- a/desktop-app/src/__tests__/chatStore.test.ts
+++ b/desktop-app/src/__tests__/chatStore.test.ts
@@ -271,32 +271,6 @@ describe('deleteChat', () => {
)
})
- it('retains legacy tombstone reads without carrying them into a new team', async () => {
- await fs.mkdir(agentPath(), { recursive: true })
- await fs.writeFile(
- agentPath('index.json'),
- JSON.stringify({
- version: 2,
- lastActiveChatId: null,
- onboardingDismissed: false,
- chats: [],
- deletedChatIds: ['legacy-chat', 'legacy-unscoped-only'],
- })
- )
- const teamBStore = new ChatStore(tempDir)
- teamBStore.setAuthorityScope(TEAM_B_SCOPE)
- expect((await teamBStore.getIndex('agent-1')).deletedChatIds).toContain('legacy-chat')
- await expect(teamBStore.createChat('agent-1', 'legacy-chat')).resolves.toMatchObject({
- id: 'legacy-chat',
- })
-
- const unscopedStore = new ChatStore(tempDir)
- unscopedStore.setAuthorityScope({ ...TEAM_A_SCOPE, teamId: null })
- await expect(unscopedStore.createChat('agent-1', 'legacy-unscoped-only')).rejects.toThrow(
- 'Chat was deleted locally'
- )
- })
-
it('keeps cleanup queued after a failure and retries it only in the owning scope', async () => {
await store.createChat('agent-1', 'retry-cleanup')
await store.saveMessages('agent-1', 'retry-cleanup', [
@@ -1826,14 +1800,12 @@ describe('corrupt/missing files', () => {
lastActiveChatId: null,
onboardingDismissed: false,
chats: [],
- deletedChatIds: [],
})
await expect(store.getIndex('agent-1')).resolves.toEqual({
version: 2,
lastActiveChatId: null,
onboardingDismissed: false,
chats: [],
- deletedChatIds: [],
})
const quarantinedIndexes = (await fs.readdir(agentPath('.corrupt'))).filter(name =>
name.startsWith('index-')
diff --git a/desktop-app/src/chatStore.ts b/desktop-app/src/chatStore.ts
index 2bba0fd82..e8b985e07 100644
--- a/desktop-app/src/chatStore.ts
+++ b/desktop-app/src/chatStore.ts
@@ -196,18 +196,12 @@ function parseChatIndex(raw: string): ChatIndex {
(candidate.lastActiveChatId !== null && typeof candidate.lastActiveChatId !== 'string') ||
typeof candidate.onboardingDismissed !== 'boolean' ||
!Array.isArray(candidate.chats) ||
- (candidate.deletedChatIds !== undefined &&
- (!Array.isArray(candidate.deletedChatIds) ||
- !candidate.deletedChatIds.every((id: unknown) => typeof id === 'string'))) ||
(candidate.deletedChatTombstones !== undefined &&
(!Array.isArray(candidate.deletedChatTombstones) ||
!candidate.deletedChatTombstones.every(isChatDeleteTombstone))) ||
(candidate.pendingChatCleanup !== undefined &&
(!Array.isArray(candidate.pendingChatCleanup) ||
- !candidate.pendingChatCleanup.every(isChatDeleteTombstone))) ||
- (candidate.pendingCleanupChatIds !== undefined &&
- (!Array.isArray(candidate.pendingCleanupChatIds) ||
- !candidate.pendingCleanupChatIds.every((id: unknown) => typeof id === 'string')))
+ !candidate.pendingChatCleanup.every(isChatDeleteTombstone)))
) {
throw new Error('Invalid chat index')
}
@@ -324,7 +318,6 @@ function emptyIndex(): ChatIndex {
lastActiveChatId: null,
onboardingDismissed: false,
chats: [],
- deletedChatIds: [],
}
}
@@ -357,20 +350,15 @@ export class ChatStore {
return current
}
- private isDeletedInScope(index: ChatIndex, chatId: string, scope = this.authorityScope): boolean {
- if (
+ private isDeletedInScope(index: ChatIndex, chatId: string): boolean {
+ return Boolean(
index.deletedChatTombstones?.some(
tombstone =>
tombstone.chatId === chatId &&
- (!scope || sameChatAuthorityScope(tombstone.authorityScope, scope))
+ (!this.authorityScope ||
+ sameChatAuthorityScope(tombstone.authorityScope, this.authorityScope))
)
- ) {
- return true
- }
- // Legacy entries were global inside an env/user store. Retain that behavior
- // for the unscoped legacy account and for stores used without authority
- // context, but never carry an unknowable old-team deletion into a new team.
- return Boolean(index.deletedChatIds?.includes(chatId) && (!scope || scope.teamId === null))
+ )
}
/**
diff --git a/desktop-app/src/types.ts b/desktop-app/src/types.ts
index 4e37addbb..78b961f30 100644
--- a/desktop-app/src/types.ts
+++ b/desktop-app/src/types.ts
@@ -936,14 +936,10 @@ export interface ChatIndex {
lastActiveChatId: string | null
onboardingDismissed: boolean
chats: ChatMetadata[]
- /** Legacy tombstones from pre-scoped versions; retained for read compatibility. */
- deletedChatIds?: string[]
/** New deletions are isolated to their complete authority identity. */
deletedChatTombstones?: ChatDeleteTombstone[]
/** Durable artifact-cleanup work, retried only in the owning authority scope. */
pendingChatCleanup?: ChatDeleteTombstone[]
- /** Legacy pending cleanup field, read for compatibility with intermediate builds. */
- pendingCleanupChatIds?: string[]
}
export interface ChatDeleteResult {
diff --git a/desktop-app/ui/src/App.tsx b/desktop-app/ui/src/App.tsx
index e5e09702f..61b024077 100644
--- a/desktop-app/ui/src/App.tsx
+++ b/desktop-app/ui/src/App.tsx
@@ -2247,6 +2247,7 @@ export function App() {
vm.notificationActionById,
vm.notifications,
vm.isHostAccessBlocked,
+ vm.hostAuthorityRevision,
vm.pendingApprovalActionId,
vm.pendingApprovals,
vm.pendingApprovalsLoading,
diff --git a/desktop-app/ui/src/__tests__/App.chatDrawer.test.tsx b/desktop-app/ui/src/__tests__/App.chatDrawer.test.tsx
index 618910fe3..e6bbe238c 100644
--- a/desktop-app/ui/src/__tests__/App.chatDrawer.test.tsx
+++ b/desktop-app/ui/src/__tests__/App.chatDrawer.test.tsx
@@ -485,7 +485,7 @@ describe('App chat drawer — reopen preserves the last-viewed chat', () => {
expect(screen.getByRole('button', { name: 'Open chats' }).textContent).toContain('Second chat')
})
- it('hides already-delivered notifications for a revoked Host', () => {
+ it('hides already-delivered notifications after a Host is revoked', () => {
const notification = {
id: 'approval-1',
kind: 'approval_required',
@@ -494,12 +494,26 @@ describe('App chat drawer — reopen preserves the last-viewed chat', () => {
timestamp: 1,
read: false,
} as AppNotification
+ let hostBlocked = false
+ let hostAuthorityRevision = 0
+ const isHostAccessBlocked = vi.fn(() => hostBlocked)
currentController = makeController({
notifications: [notification],
- isHostAccessBlocked: vi.fn(() => true),
+ isHostAccessBlocked,
+ hostAuthorityRevision,
} as Partial<AppController>)
- render(<App />)
+ const { rerender } = render(<App />)
+
+ expect(appHeaderHarness.notifications).toEqual([notification])
+ expect(appHeaderHarness.unreadNotificationCount).toBe(1)
+
+ act(() => {
+ hostBlocked = true
+ hostAuthorityRevision += 1
+ currentController.hostAuthorityRevision = hostAuthorityRevision
+ rerender(<App />)
+ })
expect(appHeaderHarness.notifications).toEqual([])
expect(appHeaderHarness.unreadNotificationCount).toBe(0)
diff --git a/desktop-app/ui/src/hooks/__tests__/useAppController.test.tsx b/desktop-app/ui/src/hooks/__tests__/useAppController.test.tsx
index 74e712324..7f8b47c0d 100644
--- a/desktop-app/ui/src/hooks/__tests__/useAppController.test.tsx
+++ b/desktop-app/ui/src/hooks/__tests__/useAppController.test.tsx
@@ -2,11 +2,24 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { scheduleAfterFirstPaint } from '../scheduleAfterFirstPaint'
import {
+ getHostAuthorityEpochForAgent,
loadWorkflowRunsWithArtifactsForApprovalRefresh,
loadWorkflowRunsWithArtifactsForWorkflowTarget,
scheduleAfterFirstPaintForSession,
} from '../useAppController'
+describe('getHostAuthorityEpochForAgent', () => {
+ it('keeps Host X chat-list authority unchanged when Host Y is revoked', () => {
+ const epochs = new Map<string, number>()
+ const hostXChatListEpoch = getHostAuthorityEpochForAgent(epochs, 'host-x')
+
+ epochs.set('host-y', 1)
+
+ expect(getHostAuthorityEpochForAgent(epochs, 'host-x')).toBe(hostXChatListEpoch)
+ expect(getHostAuthorityEpochForAgent(epochs, 'host-y')).toBe(1)
+ })
+})
+
function installWorkflowHarness(runsResult: unknown) {
const runs = vi.fn(async () => runsResult)
const listRunArtifacts = vi.fn(async () => ({
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/mockClerum.ts b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/mockClerum.ts
index f9381498e..7c349070f 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/mockClerum.ts
+++ b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/mockClerum.ts
@@ -89,7 +89,9 @@ export function installMockClerum(): MockClerum {
})),
delete: vi.fn(async () => ({ cleanupPending: false })),
loadMessages: vi.fn(async () => []),
- appendMessages: vi.fn(async () => undefined),
+ appendMessages: vi.fn(
+ async (_agentRef: string, _chatId: string, _messages: unknown[]) => undefined
+ ),
upsertMessages: vi.fn(async (agentRef: string, chatId: string, messages: unknown[]) => {
const unseen = messages.filter(message => {
const id = (message as { id?: unknown }).id
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
index a275f386c..2d1ee4713 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/chatListTitleMerge.test.tsx
@@ -63,7 +63,6 @@ function localIndex(
updatedAt: NOW,
messageCount: 0,
})),
- deletedChatIds,
deletedChatTombstones: deletedChatIds.map(chatId => ({ chatId, authorityScope })),
}
}
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/renameSessionQueue.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/renameSessionQueue.test.tsx
index 183a1ffe3..b5d8ca28c 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/renameSessionQueue.test.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/renameSessionQueue.test.tsx
@@ -54,10 +54,9 @@ function localIndex(chats: Array<{ id: string; title: string }>): ChatIndex {
}
describe('scoped deletion visibility', () => {
- it('filters tombstones by environment, user, and team while keeping legacy reads conservative', () => {
+ it('filters tombstones by environment, user, and team', () => {
const index: ChatIndex = {
...localIndex([]),
- deletedChatIds: ['legacy-delete'],
deletedChatTombstones: [
{
chatId: 'team-a-delete',
@@ -90,7 +89,7 @@ describe('scoped deletion visibility', () => {
userId: 'user-1',
teamId: null,
})
- ).toEqual(['legacy-delete'])
+ ).toEqual([])
})
})
diff --git a/desktop-app/ui/src/hooks/domain/useAgentChatController.ts b/desktop-app/ui/src/hooks/domain/useAgentChatController.ts
index 1a9dfdee4..9622afe20 100644
--- a/desktop-app/ui/src/hooks/domain/useAgentChatController.ts
+++ b/desktop-app/ui/src/hooks/domain/useAgentChatController.ts
@@ -2734,7 +2734,10 @@ export function useAgentChatController({
const sendScopeIdentity = currentAuthScopeRef.current
const originalDraftChat = activeChatVisibilityRef.current.activeChatId
const originalDraftAgent = selectedAgent
- const originalDraftRevision = getComposerDraftRevision(originalDraftChat, originalDraftAgent)
+ const originalDraftRevision = getComposerDraftRevision(
+ originalDraftChat,
+ originalDraftAgent ?? undefined
+ )
const originalAttachmentRevision = composerAttachmentRevisionRef.current
const trimmedContent = content.trim()
const effectiveAttachments = [...attachments]
@@ -2977,7 +2980,8 @@ export function useAgentChatController({
activeChatVisibilityRef.current.selectedAgent === sendAgent &&
activeChatVisibilityRef.current.activeChatId === sendChatId &&
composerAttachmentRevisionRef.current === originalAttachmentRevision &&
- getComposerDraftRevision(originalDraftChat, originalDraftAgent) === originalDraftRevision
+ getComposerDraftRevision(originalDraftChat, originalDraftAgent ?? undefined) ===
+ originalDraftRevision
) {
clearComposerAfterSend(sendChatId)
clearComposerDraft(null, sendAgent)
diff --git a/desktop-app/ui/src/hooks/domain/useChatListController.ts b/desktop-app/ui/src/hooks/domain/useChatListController.ts
index 91c0f94a5..4bca30e5b 100644
--- a/desktop-app/ui/src/hooks/domain/useChatListController.ts
+++ b/desktop-app/ui/src/hooks/domain/useChatListController.ts
@@ -175,13 +175,13 @@ function sameAuthorityScope(left: ChatAuthorityScope, right: ChatAuthorityScope)
}
export function deletedChatIdsForScope(index: ChatIndex, scope: ChatAuthorityScope): string[] {
- const scoped = (index.deletedChatTombstones ?? [])
- .filter(tombstone => sameAuthorityScope(tombstone.authorityScope, scope))
- .map(tombstone => tombstone.chatId)
- // Legacy IDs have no known team owner. Keep their historic behavior for the
- // unscoped account, but don't carry them into a different new team.
- const legacy = scope.teamId === null ? (index.deletedChatIds ?? []) : []
- return [...new Set([...legacy, ...scoped])]
+ return [
+ ...new Set(
+ (index.deletedChatTombstones ?? [])
+ .filter(tombstone => sameAuthorityScope(tombstone.authorityScope, scope))
+ .map(tombstone => tombstone.chatId)
+ ),
+ ]
}
export function useChatListController({
diff --git a/desktop-app/ui/src/hooks/useAppController.ts b/desktop-app/ui/src/hooks/useAppController.ts
index d0da62f43..b313f8a70 100644
--- a/desktop-app/ui/src/hooks/useAppController.ts
+++ b/desktop-app/ui/src/hooks/useAppController.ts
@@ -71,6 +71,13 @@ const DENY_REASON_BY_SOURCE: Record<ApprovalDecisionTarget['source'], string> =
connect_completed: 'User denied',
}
+export function getHostAuthorityEpochForAgent(
+ epochByAgent: ReadonlyMap<string, number>,
+ agentRef: string
+): number {
+ return epochByAgent.get(agentRef) ?? 0
+}
+
export async function loadWorkflowRunsWithArtifactsForWorkflowTarget(
target: Pick<WorkflowCompletionNotificationTarget, 'namespace' | 'name'>
): Promise<WorkflowRunsResult> {
@@ -324,7 +331,7 @@ export function useAppController() {
)
const getHostAuthorityEpoch = useCallback(
(agentRef: string) =>
- hostAuthorityEpochByAgentRef.current.get(agentRef) ?? hostAuthorityEpochRef.current,
+ getHostAuthorityEpochForAgent(hostAuthorityEpochByAgentRef.current, agentRef),
[]
)
useEffect(() => {
diff --git a/desktop-app/ui/src/lib/format.ts b/desktop-app/ui/src/lib/format.ts
index 693352c5a..b5ca2c31b 100644
--- a/desktop-app/ui/src/lib/format.ts
+++ b/desktop-app/ui/src/lib/format.ts
@@ -166,10 +166,14 @@ function httpErrorStatus(err: unknown): number | undefined {
const causeStatus = (err as { cause?: { status?: unknown } }).cause?.status
if (typeof causeStatus === 'number') return causeStatus
}
- // ApiError puts the response status before the body. Using the first token
- // keeps a 503 body that mentions 403 from becoming a revocation signal.
- const match = /\b([1-5]\d{2})\b/.exec(errorText(err))
- return match ? Number(match[1]) : undefined
+ // Parse only labeled HTTP statuses so digits in a host name or body aren't
+ // mistaken for the response status.
+ const match =
+ /^(?:http\s+)?(?<leading>[1-5]\d{2})\b|\b(?:http(?:\s+status)?|status(?:\s+code)?)\s*[:=]?\s*(?<labeled>[1-5]\d{2})\b|\((?<parenthesized>[1-5]\d{2})\)/i.exec(
+ errorText(err)
+ )
+ const status = match?.groups?.leading ?? match?.groups?.labeled ?? match?.groups?.parenthesized
+ return status ? Number(status) : undefined
}
/** A transcript 404 is ambiguous: absence, authority and wake failure share it. */
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 4186199f3..8c1fbebd7 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.9.0',
externalRestApiVersion: '0.1.84',
- rpcProxyVersion: '0.1.96',
+ rpcProxyVersion: '0.1.97',
desktopVersion: '0.9.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 9921b0fb3..0626fc239 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.358",
+ "version": "0.1.359",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.358",
+ "version": "0.1.359",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index a7da3f67a..d87e0b8af 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.358",
+ "version": "0.1.359",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 74f1a3511..97d69af83 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -3671,9 +3671,10 @@ export class HostReconciler {
// Preserve UID/resourceVersion and every applied field. Conflicts
// re-read this object before retrying; replicas is the only field that
// may change. An unverified legacy object must never be scaled.
+ const replicas = effective?.allowScaleUpDuringHold ? 1 : (existing.spec.replicas ?? 1)
return {
...existing,
- spec: { ...existing.spec, replicas: 1 },
+ spec: { ...existing.spec, replicas },
}
}
return this.buildDeployment(
@@ -4799,19 +4800,28 @@ export class HostReconciler {
return false
}
}
- const holdAppliedRuntime = async (): Promise<void> => {
+ const holdAppliedRuntime = async (allowScaleUp: boolean): Promise<void> => {
+ await this.provisionRuntimeTokenRevision(host, {
+ forceFreshForWake: allowScaleUp,
+ targetSuspended: lifecycle.effective.state === 'suspended',
+ })
+ revalidateHostMutationBoundary()
const applied = await this.ensureDeployment(
host,
mounts,
undefined,
- lifecycle.effective,
+ { ...lifecycle.effective, allowScaleUpDuringHold: allowScaleUp },
undefined,
revalidateHostMutationBoundary
)
revalidateHostMutationBoundary()
- this.lifecycle.markHostNotSuspended(host.name)
+ if (lifecycle.effective.state !== 'suspended' || allowScaleUp) {
+ this.lifecycle.markHostNotSuspended(host.name)
+ }
const deploymentReady =
- applied && (await this.checkDeploymentReady(host.name, host.namespace))
+ lifecycle.effective.state !== 'suspended' &&
+ applied &&
+ (await this.checkDeploymentReady(host.name, host.namespace))
revalidateHostMutationBoundary()
const ready = deploymentReady && pvcApplied && serviceApplied && npFailures.length === 0
const failures = [
@@ -4836,9 +4846,11 @@ export class HostReconciler {
? 'Waiting for CommunicationChannel inventory before creating runtime'
: failures.length > 0
? `degraded — Host runtime boundary incomplete while preserving applied runtime (${failures.join('; ')})`
- : deploymentReady
- ? 'CommunicationChannel inventory unavailable; preserving applied runtime with one replica'
- : 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
+ : lifecycle.effective.state === 'suspended' && !allowScaleUp
+ ? 'CommunicationChannel inventory unavailable; preserving suspended Host replicas'
+ : deploymentReady
+ ? 'CommunicationChannel inventory unavailable; preserving applied runtime'
+ : 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
})
}
@@ -4925,7 +4937,7 @@ export class HostReconciler {
revalidateHostMutationBoundary()
if (lifecycle.effective.suspensionBlocked) {
- await holdAppliedRuntime()
+ await holdAppliedRuntime(forceFreshForWake)
return
}
diff --git a/host-context-controller/src/statelessLifecycle.types.ts b/host-context-controller/src/statelessLifecycle.types.ts
index f63a6a5ed..6f7b098fe 100644
--- a/host-context-controller/src/statelessLifecycle.types.ts
+++ b/host-context-controller/src/statelessLifecycle.types.ts
@@ -17,6 +17,8 @@ export interface EffectiveHostLifecycle {
state: HostLifecycleState
/** Unknown channel authority: preserve the entire applied Deployment template. */
suspensionBlocked?: boolean
+ /** A fresh-read-verified wake may scale the held Deployment without a template change. */
+ allowScaleUpDuringHold?: boolean
}
/** Reconcile-time lifecycle assessment: effective mode + durable status + condition. */
diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.ts
index a6e8292b4..4647ce538 100644
--- a/host-context-controller/src/statelessLifecycleExecutor.ts
+++ b/host-context-controller/src/statelessLifecycleExecutor.ts
@@ -22,7 +22,13 @@ import {
HostLifecycleAssessment,
SuspendFromHeartbeatOutcome,
} from './statelessLifecycle.types'
-import { HostCRD, HostCondition, HostCrdStatus, HostLifecycleStatus } from './types'
+import {
+ HostCRD,
+ HostCondition,
+ HostCrdStatus,
+ HostLifecycleState,
+ HostLifecycleStatus,
+} from './types'
import { getErrorCode } from './utils'
// ─── Stateless lifecycle (Stage 2) ─────────────────────────────────────────
@@ -33,7 +39,7 @@ const PLURAL_HOSTS = 'hosts'
const STATELESS_REJECTED_CONDITION_TYPE = 'StatelessEnableRejected'
const COMMUNICATION_CHANNEL_CACHE_UNSYNCED_REASON = 'CommunicationChannelCacheUnsynced'
const COMMUNICATION_CHANNEL_CACHE_UNSYNCED_MESSAGE =
- 'CommunicationChannel cache is not synchronized; stateless lifecycle is held active'
+ 'CommunicationChannel cache is not synchronized; stateless lifecycle transitions are held'
const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'
/**
* Host CR annotation carrying the monotonic wake generation (Stage 4.3).
@@ -323,7 +329,10 @@ export class StatelessLifecycleExecutor {
}
if (!this.isCommunicationChannelCacheSynced()) {
- return this.holdActiveDuringChannelCacheRecovery(wakeHandledGeneration)
+ return this.holdActiveDuringChannelCacheRecovery(
+ wakeHandledGeneration,
+ host.status?.lifecycle?.state ?? 'active'
+ )
}
// Accepted: preserve the durable state from the CRD status — a suspended
@@ -358,12 +367,13 @@ export class StatelessLifecycleExecutor {
/** Unknown channel inventory requires preserving the applied Deployment. */
private holdActiveDuringChannelCacheRecovery(
- wakeHandledGeneration: number
+ wakeHandledGeneration: number,
+ state: HostLifecycleState = 'active'
): HostLifecycleAssessment {
return {
- effective: { stateless: true, state: 'active', suspensionBlocked: true },
+ effective: { stateless: true, state, suspensionBlocked: true },
lifecycle: {
- state: 'active',
+ state,
wakeHandledGeneration,
reason: COMMUNICATION_CHANNEL_CACHE_UNSYNCED_MESSAGE,
},
@@ -397,7 +407,10 @@ export class StatelessLifecycleExecutor {
!this.isCommunicationChannelCacheSynced() &&
assessment.condition.reason === ACTIVE_COMMUNICATION_CHANNELS_REASON
) {
- return this.holdActiveDuringChannelCacheRecovery(assessment.lifecycle.wakeHandledGeneration)
+ return this.holdActiveDuringChannelCacheRecovery(
+ assessment.lifecycle.wakeHandledGeneration,
+ assessment.effective.state
+ )
}
return assessment
}
@@ -405,7 +418,10 @@ export class StatelessLifecycleExecutor {
const { reasons, messages } = this.communicationChannelPolicyRejection(hostName)
if (reasons.length === 0) {
if (!this.isCommunicationChannelCacheSynced() && !assessment.effective.suspensionBlocked) {
- return this.holdActiveDuringChannelCacheRecovery(assessment.lifecycle.wakeHandledGeneration)
+ return this.holdActiveDuringChannelCacheRecovery(
+ assessment.lifecycle.wakeHandledGeneration,
+ assessment.effective.state
+ )
}
return assessment
}
@@ -611,13 +627,11 @@ export class StatelessLifecycleExecutor {
freshLifecycle?.wakeHandledGeneration ?? 0
)
const cachedState = host.status?.lifecycle?.state ?? 'active'
- // Confirmed rejection and channel-cache uncertainty both require an
- // active state and an explanatory reason, even when the cached state
- // was already active. A state-only discriminator would treat that
- // reason as an echo and drop it; it could also preserve a fresh drain
- // decision made just before the cache lost authority.
- const assessmentForcesActive =
- assessment.condition.status === 'True' || assessment.effective.suspensionBlocked === true
+ // Confirmed rejection requires active even when the cached state was
+ // already active. Cache uncertainty is different: preserve the fresh
+ // durable state and block scale/template transitions until authority
+ // returns.
+ const assessmentForcesActive = assessment.condition.status === 'True'
const assessmentIntendsStateOverride = assessment.lifecycle.state !== cachedState
let lifecycle: HostLifecycleStatus
if (assessmentIntendsStateOverride || assessmentForcesActive) {
@@ -1407,7 +1421,7 @@ export class StatelessLifecycleExecutor {
// Cache recovery uses the full reconcile's owned, resource-version-fenced
// replica-only update. Do not send the unfenced wake scale patch here.
- if (!this.isCommunicationChannelCacheSynced()) return false
+ if (!this.isCommunicationChannelCacheSynced()) return true
// Minimal scale patch — NOT the full buildDeployment replace. It also
// starts the pod even when the heavy body aborts early (e.g. on a
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 5e226115f..e65b11ad8 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -849,34 +849,22 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
).toHaveLength(1)
})
- it('re-reads the applied template after a scale conflict and preserves the fresh UID and revision', async () => {
+ it('preserves the applied suspended replica count during cache loss', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
- let current = structuredClone(applied)
+ const current = structuredClone(applied)
current.metadata = { ...current.metadata, uid: 'deployment-uid', resourceVersion: '41' }
const baseline = JSON.stringify(current.spec!.template)
appsApi.readNamespacedDeployment.mockImplementation(async () => structuredClone(current))
- appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
- expect(request.body.metadata.uid).toBe('deployment-uid')
- expect(request.body.metadata.resourceVersion).toBe(current.metadata!.resourceVersion)
- expect(request.body.spec.template).toEqual(current.spec!.template)
- if (current.metadata!.resourceVersion === '41') {
- current.metadata!.resourceVersion = '42'
- current.spec!.template.metadata!.annotations = { 'example.org/rollout': 'fresh' }
- throw { code: 409 }
- }
- current = structuredClone(request.body)
- return current
- })
await reconciler.reconcile(host)
- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
- expect(current.spec!.replicas).toBe(1)
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(current.spec!.replicas).toBe(0)
expect(JSON.stringify(applied.spec!.template)).toBe(baseline)
- expect(current.spec!.template.metadata!.annotations).toEqual({ 'example.org/rollout': 'fresh' })
+ expect(current.spec!.template).toEqual(applied.spec!.template)
})
it.each([
@@ -983,21 +971,22 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
- it('holds the existing template when channel authority is lost during runtime provisioning', async () => {
- let synced = true
- const host = makeStatelessHost({ status: suspendedStatus() })
+ it('refreshes runtime credentials while holding the existing template during cache loss', async () => {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
const { reconciler, appsApi, customApi } = createReconciler({
- isCommunicationChannelCacheSynced: () => synced,
+ isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
const live = persistHostDeployment(appsApi, host, applied)
- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {
- synced = false
- return runtimeTokenProvision(host)
- })
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
+ expect(provision).toHaveBeenCalledOnce()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
@@ -1005,6 +994,34 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
)
})
+ it('keeps a suspended Host at zero replicas during cache loss without a pending wake', async () => {
+ const host = makeStatelessHost({ status: suspendedStatus(3) })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
+ applied.spec!.replicas = 0
+ const live = persistHostDeployment(appsApi, host, applied)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+
+ await reconciler.reconcile(host)
+
+ expect(provision).toHaveBeenCalledWith(host, {
+ forceFreshForWake: false,
+ targetSuspended: true,
+ })
+ expect(live().spec!.replicas).toBe(0)
+ expect(live().spec!.template).toEqual(applied.spec!.template)
+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle).toMatchObject({
+ state: 'suspended',
+ wakeHandledGeneration: 3,
+ })
+ })
+
it('uses the fenced replica-only update for a pending wake during cache loss', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
host.annotations = { 'clerum.io/wake-requested': '1' }
@@ -1014,7 +1031,14 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
const live = persistHostDeployment(appsApi, host, applied)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
+ expect(provision).toHaveBeenCalledWith(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ })
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
@@ -1117,13 +1141,12 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')
expect(reconciler.getEffectiveLifecycle(host)).toMatchObject({
stateless: true,
- state: 'active',
suspensionBlocked: true,
})
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
)
- expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('active')
+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('suspended')
expect(
appsApi.replaceNamespacedDeployment.mock.calls.filter(([r]) => r.name === host.name)
).toHaveLength(0)
@@ -1134,7 +1157,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
}
})
- it('keeps the stateless template active while the CommunicationChannel cache is unsynced', async () => {
+ it('preserves a suspended Host while the CommunicationChannel cache is unsynced', async () => {
const { reconciler, appsApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
@@ -1147,13 +1170,10 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
await reconciler.reconcile(host)
const deployment = live()
- expect(deployment.spec?.replicas).toBe(1)
- expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
- expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')
- expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')
+ expect(deployment.spec?.replicas).toBe(0)
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(1)
- expect(writes[0].lifecycle?.state).toBe('active')
+ expect(writes[0].lifecycle?.state).toBe('suspended')
const condition = rejectedCondition(writes[0])
expect(condition.status).toBe('False')
expect(condition.reason).toContain('CommunicationChannelCacheUnsynced')
@@ -1179,19 +1199,18 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
- expect(provision).not.toHaveBeenCalled()
+ expect(provision).toHaveBeenCalledOnce()
const deployment = live()
- expect(deployment.spec?.replicas).toBe(1)
+ expect(deployment.spec?.replicas).toBe(0)
expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')
expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')
const writes = lifecycleStatusWrites(customApi)
expect(writes).toHaveLength(2)
expect(writes[0].lifecycle?.state).toBe('suspended')
- expect(writes[1].lifecycle).toEqual({
- state: 'active',
+ expect(writes[1].lifecycle).toMatchObject({
+ state: 'suspended',
wakeHandledGeneration: 4,
- reason: 'CommunicationChannel cache is not synchronized; stateless lifecycle is held active',
})
expect(rejectedCondition(writes[1]).reason).toBe('CommunicationChannelCacheUnsynced')
expect(rejectedCondition(writes[1]).status).toBe('False')
@@ -1457,7 +1476,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(provision).toHaveBeenCalledOnce()
})
- it('does not bootstrap new runtime credentials for active cache loss without channels', async () => {
+ it('refreshes runtime credentials for active cache loss without changing replicas', async () => {
let cacheSynced = true
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
@@ -1479,7 +1498,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
- expect(provision).not.toHaveBeenCalled()
+ expect(provision).toHaveBeenCalledOnce()
const deployment = live()
expect(deployment.spec?.replicas).toBe(1)
expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index 1b7089d8e..ec06f26a3 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.96",
+ "version": "0.1.97",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.96",
+ "version": "0.1.97",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index 8f3d77c0e..e2c973b6a 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.96",
+ "version": "0.1.97",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
index cbe9ca51e..4898ebdfd 100644
--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
@@ -351,6 +351,27 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
+ it('a wake retry that times out returns 504 instead of 502', async () => {
+ const timeout = new Error('aborted')
+ timeout.name = 'AbortError'
+ serviceMock.forwardHostMessageToHost
+ .mockRejectedValueOnce(hostDownError())
+ .mockRejectedValueOnce(timeout)
+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({
+ kind: 'active',
+ wakeGeneration: null,
+ })
+
+ const response = await postMessage(makeApp()).expect(504)
+
+ expect(response.body).toEqual({ error: 'Gateway Timeout' })
+ expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)
+ expect(serviceMock.forwardHostMessageToHost.mock.calls[1][2]).toEqual({
+ async: false,
+ timeoutMs: expect.any(Number),
+ })
+ })
+
it('a token without wake scope keeps the original full upstream timeout', async () => {
authTokenMock.verifyRpcToken.mockReturnValue({
...VALID_CLAIMS,
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index d891c22c0..f6bed2a6e 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -46,7 +46,6 @@ const RFC1123_RE = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/
// own copy (the two services share no package, so each names its own).
const SESSIONS_LIMIT_CAP = 100
const MESSAGES_LIMIT_CAP = 200
-/** A confirmed stateless host may be suspended and spend seconds failing DNS/connect. */
function isSafeUpstreamPathSegment(value: string): boolean {
return (
diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
index 5f6ac615d..007bd6d06 100644
--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
@@ -5,7 +5,7 @@ import {
} from '../mcpHostRestService.js'
describe('host message availability probe', () => {
- it('aborts at the requested short timeout with a distinct wake-eligible reason', async () => {
+ it('aborts at the requested short timeout with the standard abort reason', async () => {
vi.useFakeTimers()
const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(
(_input, init) =>
@@ -20,7 +20,7 @@ describe('host message availability probe', () => {
{ timeoutMs: 750 }
)
const rejection = expect(pending).rejects.toMatchObject({
- name: 'HostAvailabilityProbeTimeoutError',
+ name: 'AbortError',
})
await vi.advanceTimersByTimeAsync(750)
await rejection
diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
index 85fe77723..29265d94e 100644
--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
@@ -664,6 +664,22 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
expect(res.statusCode).toBe(200)
})
+ it('skips an admission retry when less than 100 ms remains', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
+ const res = makeRes()
+ const attemptUpstream = vi.fn(async (_timeoutMs: number) => {})
+ const pending = respondWithWakeAndHold({
+ ...respondOptions(coordinator, res, attemptUpstream),
+ deadlineMs: Date.now() + 99,
+ })
+
+ await pending
+
+ expect(attemptUpstream).not.toHaveBeenCalled()
+ expect(res.statusCode).toBe(503)
+ })
+
it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
diff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.ts
index a1902149e..8646c6263 100644
--- a/rpc-proxy/src/services/mcpHostRestService.ts
+++ b/rpc-proxy/src/services/mcpHostRestService.ts
@@ -141,13 +141,7 @@ export async function forwardHostMessageToHost(
const query = options?.async ? '?async=true' : ''
const abortController = new AbortController()
const timeout = setTimeout(() => {
- if (options?.timeoutMs !== undefined) {
- const error = new Error('Initial host availability probe timed out')
- error.name = 'HostAvailabilityProbeTimeoutError'
- abortController.abort(error)
- } else {
- abortController.abort()
- }
+ abortController.abort()
}, options?.timeoutMs ?? config.upstreamTimeoutMs)
try {
const response = await fetch(`${baseUrl}/v1/runtime/messages${query}`, {
diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.ts
index fd6d67180..2b1cdf73a 100644
--- a/rpc-proxy/src/services/wakeAndHold.ts
+++ b/rpc-proxy/src/services/wakeAndHold.ts
@@ -45,6 +45,7 @@ const ENTRY_TTL_MARGIN_MS = 5_000
const MAX_TRACKED_WAKE_COORDINATIONS = 1_000
const WAKE_SCOPE = 'host:wake:write' as const
const ADMISSION_RETRY_DELAYS_MS = [250, 500, 1_000]
+const MIN_ADMISSION_RETRY_TIMEOUT_MS = 100
/** Leaves room for one final upstream timeout before Desktop's 60s deadline. */
export const MAX_REQUEST_HOLD_MS = 48_000
@@ -140,7 +141,7 @@ export function wakeCoordinationKey(claims: RpcAccessClaims, hostRef: string): s
* hold budget so a token that cannot survive the wake call is not treated as
* usable for it.
*/
-export function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {
+function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {
const tokenExpMs = claims.exp * 1000
if (tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= now) return false
if (!claims.scopes.includes(WAKE_SCOPE)) return false
@@ -673,7 +674,7 @@ export async function respondWithWakeAndHold(
}
try {
const remainingMs = deadlineMs - Date.now()
- if (remainingMs <= 0) break
+ if (remainingMs < MIN_ADMISSION_RETRY_TIMEOUT_MS) break
await options.attemptUpstream(Math.min(config.upstreamTimeoutMs, remainingMs))
return
} catch (error) {