Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore: merge latest dev into password admission PR
Josué92bc129e9204
diff --cc control-api/package-lock.jsonindex fe86322be,1c36f4bae..5b1803ccc--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@@ -1,12 -1,12 +1,12 @@@{"name": "clerum-control-api",- "version": "0.1.783",- "version": "0.1.755",++ "version": "0.1.784","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.783",- "version": "0.1.755",++ "version": "0.1.784","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --cc control-api/package.jsonindex 4faa2977a,b8e823509..7a3d9faae--- a/control-api/package.json+++ b/control-api/package.json@@@ -1,6 -1,6 +1,6 @@@{"name": "clerum-control-api",- "version": "0.1.783",- "version": "0.1.755",++ "version": "0.1.784","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --cc control-api/src/routes/admin/registry.tsindex 8ec9e9f5f,3fb72180c..5de7af708--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@@ -499,8 -498,8 +498,7 @@@ export function registryErrorLogFields(}type CredentialPayloadValidation =-- | { ok: true; secretData: Record<string, string> }-- | { ok: false; body: Record<string, unknown> }++ { ok: true; secretData: Record<string, string> } | { ok: false; body: Record<string, unknown> }function looksLikeCredentialPlaceholder(value: string): boolean {const normalized = value.trim().toLowerCase()@@@ -1237,11 -1236,11 +1235,7 @@@ async function readSecretMutationOutcom}export type CreatedSecretAfterDeleteFailure =-- | 'gone'-- | 'replaced'-- | 'modified-original'-- | 'unchanged-original'-- | 'identity-unavailable'++ 'gone' | 'replaced' | 'modified-original' | 'unchanged-original' | 'identity-unavailable'/** Classify what remains under a Secret name after an identity-bound delete failed. */export function classifyCreatedSecretAfterDeleteFailure(@@@ -1308,11 -1307,11 +1302,7 @@@ async function waitForSecretDeletionOrR}type CreatedResourceAfterDeleteFailure =-- | 'gone'-- | 'replaced'-- | 'modified-original'-- | 'unchanged-original'-- | 'identity-unavailable'++ 'gone' | 'replaced' | 'modified-original' | 'unchanged-original' | 'identity-unavailable'function normalizeRegistryResourceSnapshot(raw: unknown,@@@ -2112,9 -2111,9 +2102,7 @@@ export function createAdminRegistryRout}const transport = (entry.transport ?? 'streamableHttp') as-- | 'streamableHttp'-- | 'sse'-- | 'stdio'++ 'streamableHttp' | 'sse' | 'stdio'const port = (meta?.port as number | undefined) ?? 3000const secretName = `${serverName}-credentials`@@@ -5234,9 -5233,9 +5222,7 @@@const limit = toPageNum(req.query.limit)const offset = toPageNum(req.query.offset)const raw = (await listOrgEntries(ctx.orgName, { limit, offset })) as-- | { data?: unknown[]; entries?: unknown[]; meta?: unknown }-- | null-- | undefined++ { data?: unknown[]; entries?: unknown[]; meta?: unknown } | null | undefinedconst data = raw?.data ?? raw?.entries ?? []res.status(200).json(raw?.meta !== undefined ? { data, meta: raw.meta } : { data })} catch (err) {diff --cc control-api/test/routes.registryInstall.test.tsindex 80521a759,a5fc333dd..9f1c02f6e--- a/control-api/test/routes.registryInstall.test.ts+++ b/control-api/test/routes.registryInstall.test.ts@@@ -5703,8 -5703,8 +5703,7 @@@ describe('authHeaders template on remot// 3. The actual secret values landed in the K8s Secret only.const secretCall = secretSpy.mock.calls[0]?.[0] as-- | { name?: string; stringData?: Record<string, string> }-- | undefined++ { name?: string; stringData?: Record<string, string> } | undefinedexpect(secretCall?.name).toBe('sentry-separation-credentials')expect(secretCall?.stringData).toEqual({SENTRY_AUTH_TOKEN: 'super-secret-token-value',diff --cc control-ui/lib/api.tsindex 6747337f8,85edb56c1..f8ba5ee93--- a/control-ui/lib/api.ts+++ b/control-ui/lib/api.ts@@@ -668,18 -669,18 +669,13 @@@ export type GfsOperatorSubjectInput =export type GfsContextSubjectInput = { type: 'context'; id: string }export type GfsBulkGrantSubjectInput =-- | GfsUserSubjectInput-- | GfsTeamSubjectInput-- | GfsHostSubjectInput++ GfsUserSubjectInput | GfsTeamSubjectInput | GfsHostSubjectInputexport type GfsBulkShareSubjectInput = GfsUserSubjectInput | GfsTeamSubjectInputexport type GfsSubjectInput =-- | GfsBulkGrantSubjectInput-- | GfsOperatorSubjectInput-- | GfsContextSubjectInput++ GfsBulkGrantSubjectInput | GfsOperatorSubjectInput | GfsContextSubjectInputtype GfsSubjectMutationInput<TBulkSubject extends GfsSubjectInput> =-- | { subject: GfsSubjectInput; subjects?: never }-- | { subject?: never; subjects: TBulkSubject[] }++ { subject: GfsSubjectInput; subjects?: never } | { subject?: never; subjects: TBulkSubject[] }type GfsMutationRequestBody = {drive?: string@@@ -1831,11 -1832,11 +1827,7 @@@ export async function createMcpServer(p/** Cleanup stages control-api reports as `pending` when an mcp-server uninstall stops. */export type McpServerUninstallStage =-- | 'contexts'-- | 'dynamic_client'-- | 'oauth_grants'-- | 'secrets'-- | 'mcp_server'++ 'contexts' | 'dynamic_client' | 'oauth_grants' | 'secrets' | 'mcp_server'const MCP_SERVER_UNINSTALL_STAGE_LABELS: Record<McpServerUninstallStage, string> = {contexts: 'agent access',@@@ -2005,9 -2006,9 +1997,7 @@@ export async function updateMcpSecret(n}export type RecipeSecretOwnership =-- | { kind: 'shared' }-- | { kind: 'owner-recipe'; recipeName: string }-- | { kind: 'unlabeled' }++ { kind: 'shared' } | { kind: 'owner-recipe'; recipeName: string } | { kind: 'unlabeled' }export type RecipeSecretItem = {name: string@@@ -3014,13 -3015,13 +3004,7 @@@ export type WorkflowRecipePhase| 'rollback-failed'export type WorkflowExecutionPhase =-- | 'pending'-- | 'initializing'-- | 'running'-- | 'completed'-- | 'failed'-- | 'cancelled'-- | 'recovering'++ 'pending' | 'initializing' | 'running' | 'completed' | 'failed' | 'cancelled' | 'recovering'export type WorkflowRecipeStatus = {phase?: WorkflowRecipePhase@@@ -3829,10 -3839,10 +3822,7 @@@ export async function installRecipeFrom// capability ceiling, and the may_deny ⇒ fail-closed rule; error bodies arrive as// { error, reason? } (surfaced via the thrown Error's `.body`/`.code`).export type HookCapability =-- | 'may_deny'-- | 'may_rewrite'-- | 'may_substitute_result'-- | 'may_add_context'++ 'may_deny' | 'may_rewrite' | 'may_substitute_result' | 'may_add_context'export type InstallHookFromRegistryRequest = {hostRef: string@@@ -4031,8 -4041,8 +4021,7 @@@ export async function listOrgGrants(pluexport async function createOrgGrant(input: CreateOrgGrantInput): Promise<OrgGrant> {const raw = (await registryCodedRequest('POST', '/api/v1/admin/registry/grants', input)) as-- | RawOrgGrant-- | undefined++ RawOrgGrant | undefinedreturn normalizeOrgGrant(raw ?? {})}@@@ -4350,12 -4360,12 +4339,7 @@@ export type PluginWorkloadSdkQuotaCount}export type PluginWorkloadSdkInvocationStatus =-- | 'in_progress'-- | 'complete'-- | 'failed'-- | 'provider_unavailable'-- | 'accepted'-- | 'delivered'++ 'in_progress' | 'complete' | 'failed' | 'provider_unavailable' | 'accepted' | 'delivered'export type PluginWorkloadSdkInvocation = {id: string@@@ -4576,12 -4586,12 +4560,7 @@@ export async function listOrgImages():// registry_signing_material_unavailable.export type RegistryConnectionState =-- | 'disconnected'-- | 'pending'-- | 'connecting'-- | 'approved'-- | 'rejected'-- | 'connected'++ 'disconnected' | 'pending' | 'connecting' | 'approved' | 'rejected' | 'connected'export type RegistryRecoveryError =| 'already_claimed'diff --cc external-rest-api/package-lock.jsonindex 0a01f956f,dd9f4c6a3..4605405e5--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@@ -1,12 -1,12 +1,12 @@@{"name": "clerum-external-rest-api",- "version": "0.1.130",- "version": "0.1.101",++ "version": "0.1.131","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.130",- "version": "0.1.101",++ "version": "0.1.131","dependencies": {"@clerum/jwt-key-policy": "file:../packages/jwt-key-policy","cors": "^2.8.5",diff --cc external-rest-api/package.jsonindex cd2c73e1f,4b1a80ddc..f87132466--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@@ -1,6 -1,6 +1,6 @@@{"name": "clerum-external-rest-api",- "version": "0.1.130",- "version": "0.1.101",++ "version": "0.1.131","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --cc external-rest-api/src/releaseManifest.tsindex e15863de5,9ef40b16b..7e4e0f581--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@@ -8,7 -8,7 +8,7 @@@ export type ReleaseManifest =export const releaseManifest: ReleaseManifest = {releaseId: 'v0.11.0',- externalRestApiVersion: '0.1.130',- externalRestApiVersion: '0.1.101',++ externalRestApiVersion: '0.1.131',rpcProxyVersion: '0.1.126',desktopVersion: '0.11.0',minimumDesktopVersion: '0.1.252',diff --cc host-context-controller/src/config.tsindex fd7697a4b,696678ac4..16812ebfd--- a/host-context-controller/src/config.ts+++ b/host-context-controller/src/config.ts@@@ -673,18 -693,18 +693,11 @@@ export const config: Config ='us-central1-docker.pkg.dev/your-gcp-project/clerum/mcp-host:0.6.0')!,hostImagePullPolicy: getEnv('CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY', 'Always') as-- | 'Always'-- | 'IfNotPresent'-- | 'Never',++ 'Always' | 'IfNotPresent' | 'Never',statelessImagePullPolicy: getEnv('CONTEXT_MAPPER_STATELESS_IMAGE_PULL_POLICY', '') as-- | ''-- | 'Always'-- | 'IfNotPresent'-- | 'Never',++ '' | 'Always' | 'IfNotPresent' | 'Never',mcpServerImagePullPolicy: getEnv('CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY', 'IfNotPresent') as-- | 'Always'-- | 'IfNotPresent'-- | 'Never',++ 'Always' | 'IfNotPresent' | 'Never',hostImagePullSecretName: getEnv('CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET', 'clerum')!,hostPort: getEnvInt('CONTEXT_MAPPER_HOST_PORT', 8080),hostConfigMapName: getEnv('CONTEXT_MAPPER_HOST_CONFIGMAP_NAME', 'mcp-host-config')!,@@@ -729,9 -749,9 +742,7 @@@'us-central1-docker.pkg.dev/your-gcp-project/clerum/workspace-files-controller:0.1.0')!,wfcImagePullPolicy: getEnv('CONTEXT_MAPPER_WFC_IMAGE_PULL_POLICY', 'IfNotPresent') as-- | 'Always'-- | 'IfNotPresent'-- | 'Never',++ 'Always' | 'IfNotPresent' | 'Never',wfcImagePullSecretName: getEnv('CONTEXT_MAPPER_WFC_IMAGE_PULL_SECRET', 'clerum')!,wfcPort: getEnvInt('CONTEXT_MAPPER_WFC_PORT', 8086),wfcInitImage: getEnv('CONTEXT_MAPPER_WFC_INIT_IMAGE', 'busybox:1.36')!,diff --cc host-context-controller/src/types.tsindex 1886a2c15,41d37fc5d..21457efff--- a/host-context-controller/src/types.ts+++ b/host-context-controller/src/types.ts@@@ -258,11 -258,11 +258,7 @@@ export interface SharedFileSystemSpec}export type SharedFileSystemPhase =-- | 'Provisioning'-- | 'Initializing'-- | 'Ready'-- | 'Degraded'-- | 'Failed'++ 'Provisioning' | 'Initializing' | 'Ready' | 'Degraded' | 'Failed'export interface SharedFileSystemCondition {type: string@@@ -330,11 -330,11 +326,7 @@@ export interface GlobalFileSystemSpec}export type GlobalFileSystemPhase =-- | 'Provisioning'-- | 'Initializing'-- | 'Ready'-- | 'Degraded'-- | 'Failed'++ 'Provisioning' | 'Initializing' | 'Ready' | 'Degraded' | 'Failed'export interface GlobalFileSystemCondition {type: string@@@ -746,10 -748,10 +740,7 @@@ export interface LlmHookTargetexport type LlmHookLifecyclePoint = 'preCall' | 'moderate' | 'postCallSuccess' | 'onError'export type LlmHookCapability =-- | 'may_deny'-- | 'may_rewrite'-- | 'may_substitute_result'-- | 'may_add_context'++ 'may_deny' | 'may_rewrite' | 'may_substitute_result' | 'may_add_context'export interface LlmHookSpec {target: LlmHookTargetdiff --cc host-context-controller/test/hostReconciler.lifecycle.test.tsindex ef84ce9ea,18917b62e..80b150000--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@@ -378,8 -381,8 +381,7 @@@ async function mintedRuntimeCredentialRcoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })await (producer as any).ensureMcpHostRuntimeTokenSecret(host)const created = coreApi.createNamespacedSecret.mock.calls.at(-1)?.[0].body as-- | k8s.V1Secret-- | undefined++ k8s.V1Secret | undefinedif (!created?.stringData) throw new Error('producer pass wrote no runtime credential Secret')let record: k8s.V1Secret = {...created,@@@ -404,8 -407,8 +406,7 @@@coreApi.readNamespacedSecret.mockResolvedValue(record)await (producer as any).ensureMcpHostRuntimeTokenSecret(host)const consumed = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as-- | k8s.V1Secret-- | undefined++ k8s.V1Secret | undefinedif (consumed?.metadata?.annotations?.['clerum.io/runtime-token-bootstrap-state'] !== 'consumed') {diff --cc mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.tsindex 484df5044,fe5ccc0c3..c2cc8d641--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts@@@ -97,4 -184,683 +184,686 @@@ describe('PersistQueue', () =>worker.emit('exit', 137)await expect(promise).rejects.toThrow(/db worker exited/)})++ it('rejects async writes admitted after the worker exited instead of dropping them', async () => {+ const worker = new FakeWorker()+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ // Liveness witness: while the worker is alive the async write is posted.+ expect(worker.sent).toHaveLength(1)++ worker.emit('exit', 137)+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(/db worker exited \(code=137\)/)+ expect(worker.sent).toHaveLength(1)+ })++ it('fails the next write on the same queue after an async write failed', async () => {+ const worker = new FakeWorker()+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ // Witness: a sync write before any failure resolves.+ const first = q.enqueueSync({ kind: 'ping' })+ worker.emit('message', {+ id: worker.sent[0].id,+ ok: true,+ result: 'pong',+ } satisfies WorkerReply)+ await expect(first).resolves.toBe('pong')++ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ worker.emit('message', {+ id: worker.sent[1].id,+ ok: false,+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },+ } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))++ await expect(q.enqueueSync({ kind: 'ping' })).rejects.toThrow(+ /async write failed \(sessionKey=s1\): database or disk is full/+ )+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow(/database or disk is full/)+ })++ it('calls onExit and onFailure once when the worker exits unexpectedly', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const exits: number[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ onFailure: err => failures.push(err),+ onExit: code => exits.push(code),+ })+ // Liveness witness: with the worker alive a write is acknowledged and+ // onFailure has not fired.+ const ok = q.enqueueSync({ kind: 'ping' })+ worker.emit('message', {+ id: worker.sent[0].id,+ ok: true,+ result: 'pong',+ } satisfies WorkerReply)+ await expect(ok).resolves.toBe('pong')+ expect(failures).toHaveLength(0)++ worker.emit('exit', 1)+ expect(exits).toEqual([1])+ expect(failures.map(e => e.message)).toEqual(['db worker exited (code=1)'])+ // A later rejected write does not report a second failure.+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(/db worker exited/)+ expect(failures).toHaveLength(1)+ })++ it('calls onFailure once when an async write fails', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ onFailure: err => failures.push(err),+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ q.enqueueAsync('s2', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(2)++ // Liveness witness: an acknowledged async write does not fire onFailure.+ worker.emit('message', {+ id: worker.sent[0].id,+ ok: true,+ result: 'pong',+ } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))+ expect(failures).toHaveLength(0)++ worker.emit('message', {+ id: worker.sent[1].id,+ ok: false,+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },+ } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))+ expect(failures.map(e => e.message)).toEqual([+ 'async write failed (sessionKey=s2): database or disk is full',+ ])++ // A worker exit after the failure does not report a second one.+ worker.emit('exit', 1)+ expect(failures).toHaveLength(1)+ })++ it('keeps a slow async write pending past asyncTimeoutMs and its key ordered until the ACK', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 20,+ stallTimeoutMs: 30_000,+ onFailure: err => failures.push(err),+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ // Liveness witness: the first write was posted.+ expect(worker.sent).toHaveLength(1)+ await new Promise(r => setTimeout(r, 80))+ // Past the threshold: still pending, the second write still queued behind it.+ expect(failures).toHaveLength(0)+ expect(worker.sent).toHaveLength(1)++ worker.emit('message', { id: worker.sent[0].id, ok: true, result: null } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))+ expect(worker.sent).toHaveLength(2)+ worker.emit('message', { id: worker.sent[1].id, ok: true, result: null } satisfies WorkerReply)+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)+ // A later write on the same key is still admitted.+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(3)+ })++ it('reports neither onFailure nor onExit for the worker exit that follows a shutdown ACK', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const reportedExits: number[] = []+ const observedExits: number[] = []+ worker.on('exit', code => observedExits.push(code))+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ onFailure: err => failures.push(err),+ onExit: code => reportedExits.push(code),+ })+ const closing = q.close()+ await new Promise(r => setImmediate(r))+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')+ expect(shutdown).toBeDefined()+ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)+ // FakeWorker.terminate() emits the exit.+ await closing+ // Liveness witness: the exit was observed.+ expect(observedExits).toEqual([0])+ expect(reportedExits).toHaveLength(0)+ expect(failures).toHaveLength(0)+ })++ it('waits for the worker shutdown ACK before terminating it', async () => {+ const worker = new FakeWorker()+ let terminateCalls = 0+ worker.terminate = () => {+ terminateCalls += 1+ }+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ const closing = q.close()+ await new Promise(r => setImmediate(r))++ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')+ // Witness: the shutdown op was posted.+ expect(shutdown).toBeDefined()+ expect(terminateCalls).toBe(0)++ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)+ await closing+ expect(terminateCalls).toBe(1)+ })++ it('drains queued async writes before posting shutdown and rejects close on a failed one', async () => {+ const worker = new FakeWorker()+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ const closing = q.close()+ await new Promise(r => setImmediate(r))+ // The queued write is still in flight, so shutdown is not posted yet.+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])++ worker.emit('message', {+ id: worker.sent[0].id,+ ok: false,+ error: { code: 'SQLITE_IOERR', message: 'disk I/O error' },+ } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')+ expect(shutdown).toBeDefined()+ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)+ await expect(closing).rejects.toThrow(/disk I\/O error/)+ })++ it('never posts a write chained behind a failed async write on the same key', async () => {+ const worker = new FakeWorker()+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ q.enqueueAsync('s1', { kind: 'ping' })+ const chainedSync = q.enqueueSync({ kind: 'ping' }, 's1')+ await Promise.resolve()+ // Liveness witness: the first write was posted; the others wait on it.+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])++ worker.emit('message', {+ id: worker.sent[0].id,+ ok: false,+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },+ } satisfies WorkerReply)+ await expect(chainedSync).rejects.toThrow(+ /async write failed \(sessionKey=s1\): database or disk is full/+ )+ await new Promise(r => setImmediate(r))+ expect(worker.sent).toHaveLength(1)+ })++ it('resolves a concurrent second close only after the shutdown ACK', async () => {+ const worker = new FakeWorker()+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })+ let secondSettled = false+ const first = q.close()+ const second = q.close().finally(() => {+ secondSettled = true+ })+ await new Promise(r => setImmediate(r))+ const shutdowns = worker.sent.filter(m => m.op.kind === 'shutdown')+ // Witness: exactly one shutdown was posted, and it is not ACKed yet.+ expect(shutdowns).toHaveLength(1)+ expect(secondSettled).toBe(false)++ worker.emit('message', { id: shutdowns[0].id, ok: true, result: null } satisfies WorkerReply)+ await Promise.all([first, second])+ expect(secondSettled).toBe(true)+ })++ it('rejects close when the worker reports a shutdown failure', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ onFailure: err => failures.push(err),+ })+ const closing = q.close()+ await new Promise(r => setImmediate(r))+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')+ expect(shutdown).toBeDefined()+ worker.emit('message', {+ id: shutdown!.id,+ ok: false,+ error: { code: 'SQLITE_BUSY', message: 'database is locked' },+ } satisfies WorkerReply)+ await expect(closing).rejects.toThrow(/database is locked/)+ // The shutdown failure is the queue's failure; the exit that terminate()+ // causes afterwards is not reported as a second one.+ expect(failures.map(e => e.message)).toEqual(['database is locked'])+ })+ })++ // A slow worker and a wedged worker differ in progress, not in how long a+ // write waits: a slow worker keeps completing operations, a wedged one stops.+ // These cases pin both, in service and at close(), so a change that fixes one+ // by breaking another fails here.+ describe('PersistQueue progress bound: slow vs wedged, in service and at close()', () => {+ const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))+ const ack = (worker: FakeWorker, msg: WorkerMessage) =>+ worker.emit('message', { id: msg.id, ok: true, result: null } satisfies WorkerReply)++ /** A queue whose open ping was answered, so the progress bound is armed. */+ async function openQueue(opts: {+ stallTimeoutMs: number+ asyncTimeoutMs?: number+ syncTimeoutMs?: number+ }) {+ const worker = new FakeWorker()+ const failures: Error[] = []+ let terminateCalls = 0+ worker.terminate = () => {+ terminateCalls += 1+ worker.emit('exit', 1)+ }+ const q = new PersistQueue(worker, {+ syncTimeoutMs: opts.syncTimeoutMs ?? 5000,+ asyncTimeoutMs: opts.asyncTimeoutMs ?? 5000,+ stallTimeoutMs: opts.stallTimeoutMs,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen()+ ack(worker, worker.sent[0])+ await open+ return { worker, q, failures, terminateCalls: () => terminateCalls }+ }++ /**+ * Answers every posted operation in order, one at a time, `delayMs` after+ * the previous answer: a single-writer worker that is slow but alive.+ */+ function autoAck(worker: FakeWorker, delayMs: number): WorkerMessage[] {+ const acked: WorkerMessage[] = []+ let busy = false+ const queue: WorkerMessage[] = []+ const next = () => {+ const msg = queue.shift()+ if (!msg) {+ busy = false+ return+ }+ busy = true+ setTimeout(() => {+ acked.push(msg)+ ack(worker, msg)+ next()+ }, delayMs)+ }+ const post = worker.postMessage.bind(worker)+ worker.postMessage = msg => {+ post(msg)+ queue.push(msg)+ if (!busy) next()+ }+ return acked+ }++ it('fails a wedged worker in service within the bound and releases everything queued behind it', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({+ stallTimeoutMs: 100,+ asyncTimeoutMs: 20,+ })+ // Liveness witness: a write on the same key is acknowledged first.+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ ack(worker, worker.sent[1])+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)++ // The worker stops answering.+ q.enqueueAsync('s1', { kind: 'ping' })+ const keyedSync = q.enqueueSync({ kind: 'ping' }, 's1')+ await Promise.resolve()+ expect(worker.sent).toHaveLength(3)+ const started = Date.now()+ const prefixDrained = q.drainPrefix('s')++ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(Date.now() - started).toBeLessThan(400)+ expect(failures[0].message).toMatch(/db worker made no progress for 100ms/)+ await expect(keyedSync).rejects.toBe(failures[0])+ await prefixDrained+ expect(terminateCalls()).toBe(1)+ // The write chained behind the stalled one was never posted.+ expect(worker.sent).toHaveLength(3)+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow(failures[0].message)+ })++ it('never fails a slow but alive worker with a backlog far longer than the bound', async () => {+ // A 10x margin between the ACK gap and the bound keeps this stable under load.+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 500, asyncTimeoutMs: 20 })+ const acked = autoAck(worker, 50)+ const keys = ['a', 'b', 'c']+ // The tag only identifies the write in this test; the queue never reads it.+ const tagged = (tag: string) => ({ kind: 'ping', tag }) as WorkerOp+ for (let i = 0; i < 10; i++) {+ for (const key of keys) q.enqueueAsync(key, tagged(`${key}${i}`))+ }+ await Promise.all(keys.map(k => q.drainSessionKey(k)))+ const tags = acked.map(m => (m.op as WorkerOp & { tag: string }).tag)+ // Liveness witness: all 30 writes were acknowledged, 1.5 s of backlog+ // against a 500 ms bound, each key in its enqueue order.+ expect(tags).toHaveLength(30)+ for (const key of keys) {+ expect(tags.filter(t => t.startsWith(key))).toEqual(+ Array.from({ length: 10 }, (_, i) => `${key}${i}`)+ )+ }+ expect(failures).toHaveLength(0)+ })++ it('drains a slow backlog completely on close() before posting shutdown', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, asyncTimeoutMs: 50 })+ const acked = autoAck(worker, 10)+ for (let i = 0; i < 10; i++) q.enqueueAsync('s1', { kind: 'ping' })+ await q.close()+ const kinds = acked.map(m => m.op.kind)+ // Liveness witness: the ten writes and then shutdown were acknowledged.+ expect(kinds).toEqual([...Array(10).fill('ping'), 'shutdown'])+ expect(failures).toHaveLength(0)+ })++ it('drains on close() a backlog longer than twice the bound, with no total deadline', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 500 })+ const acked = autoAck(worker, 50)+ for (let i = 0; i < 30; i++) q.enqueueAsync('s1', { kind: 'ping' })+ const started = Date.now()+ await expect(q.close()).resolves.toBeUndefined()+ // Liveness witness: 30 writes, then shutdown, all acknowledged; the+ // backlog (30 x 50 ms) is longer than 2x the 500 ms bound.+ expect(acked.map(m => m.op.kind)).toEqual([...Array(30).fill('ping'), 'shutdown'])+ expect(Date.now() - started).toBeGreaterThan(1000)+ expect(failures).toHaveLength(0)+ })++ it('does not count idle time against the bound on the next write', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 200 })+ // Idle for twice the bound with nothing in flight.+ await sleep(400)+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(2)+ await sleep(20)+ ack(worker, worker.sent[1])+ // Liveness witness: the write was posted and its ACK drained the key.+ await q.drainSessionKey('s1')+ expect(worker.sent[1].op.kind).toBe('ping')+ await sleep(50)+ expect(failures).toHaveLength(0)+ })++ it('fails close() within the bound when the worker is wedged', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })+ // Liveness witness: a write is acknowledged before the worker stops.+ const first = q.enqueueSync({ kind: 'ping' })+ ack(worker, worker.sent[1])+ await first+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ const started = Date.now()+ await expect(q.close()).rejects.toThrow(/db worker made no progress for 100ms/)+ expect(Date.now() - started).toBeLessThan(400)+ expect(terminateCalls()).toBe(1)+ expect(failures).toHaveLength(1)+ expect(worker.sent.map(m => m.op.kind)).not.toContain('shutdown')+ })++ it('waits on close() for a sync write without a sessionKey before posting shutdown', async () => {+ const { worker, q } = await openQueue({ stallTimeoutMs: 1000 })+ const unkeyed = q.enqueueSync<string>({ kind: 'ping' })+ const closing = q.close()+ await sleep(20)+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping', 'ping'])++ worker.emit('message', {+ id: worker.sent[1].id,+ ok: true,+ result: 'pong',+ } satisfies WorkerReply)+ // Liveness witness: the sync write resolves with the worker's result.+ await expect(unkeyed).resolves.toBe('pong')+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))+ ack(worker, worker.sent[2])+ await closing+ })++ it('does not fail a healthy worker whose answer waited behind a main-thread stall', async () => {+ const realWorker = new Worker(+ `const { parentPort } = require('node:worker_threads')+ parentPort.on('message', msg => setTimeout(() => parentPort.postMessage({ id: msg.id, ok: true, result: null }), 10))`,+ { eval: true }+ )+ const failures: Error[] = []+ const q = new PersistQueue(realWorker as unknown as WorkerLike, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 100,+ onFailure: err => failures.push(err),+ })+ try {+ await q.awaitOpen()+ const write = q.enqueueSync({ kind: 'ping' }, 's1')+ await sleep(1)+ // The main thread is blocked well past the bound while the answer+ // waits in the port. Blocking in the check phase makes the next loop+ // iteration run the overdue timer before the poll phase reads the+ // answer (blocking in the timers phase would not: libuv compares+ // timers against the loop time cached when that phase started).+ await new Promise<void>(resolve =>+ setImmediate(() => {+ const until = Date.now() + 300+ while (Date.now() < until) {+ /* block the event loop */+ }+ resolve()+ })+ )+ // Liveness witness: the write resolves.+ await expect(write).resolves.toBeNull()+ await sleep(150)+ expect(failures).toHaveLength(0)+ } finally {+ await realWorker.terminate()+ }+ })++ it('does not start the bound before the database is open', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 100,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen()+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(2)+ await sleep(300)+ expect(failures).toHaveLength(0)++ ack(worker, worker.sent[0])+ await open+ ack(worker, worker.sent[1])+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)++ // Liveness witness: once open, a wedged write does fail the queue.+ q.enqueueAsync('s1', { kind: 'ping' })+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(failures[0].message).toMatch(/no progress for 100ms/)+ })++ it('fails close() within the bound when the worker never answers the open ping', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 100,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen().catch((err: unknown) => err)+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ // Both are posted before close(), so nothing sent after it can arm the bound.+ expect(worker.sent).toHaveLength(2)+ const closing = q.close()+ let settled = false+ const outcome = closing.then(+ () => ({ ok: true as const }),+ (err: Error) => ({ ok: false as const, err })+ )+ void outcome.then(() => {+ settled = true+ })+ const started = Date.now()+ // Bounded wait, so a hang fails the assertion instead of the test timeout.+ await Promise.race([outcome, sleep(500)])+ // Liveness witnesses: the open ping was posted, and close() started.+ expect(worker.sent[0].op.kind).toBe('ping')+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow()+ expect(settled).toBe(true)+ const result = await outcome+ expect(result.ok).toBe(false)+ expect(result.ok ? null : result.err.message).toMatch(/db worker made no progress for 100ms/)+ expect(Date.now() - started).toBeLessThan(500)+ expect(failures).toHaveLength(1)+ expect(await open).toBe(failures[0])+ })++ it('lets a slow but healthy open finish during close()', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ let terminateCalls = 0+ worker.terminate = () => {+ terminateCalls += 1+ worker.emit('exit', 0)+ }+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 300,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen()+ q.enqueueAsync('s1', { kind: 'ping' })+ // The open takes longer than the bound before close() starts; the bound+ // counts from close(), not from the ping.+ await sleep(400)+ expect(worker.sent).toHaveLength(2)+ const closing = q.close()+ await sleep(30)+ ack(worker, worker.sent[0])+ await open+ ack(worker, worker.sent[1])+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))+ // Liveness witness: the async write was acknowledged and shutdown posted.+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping', 'ping', 'shutdown'])+ ack(worker, worker.sent[2])+ await expect(closing).resolves.toBeUndefined()+ expect(failures).toHaveLength(0)+ expect(terminateCalls).toBe(1)+ })++ it('stays failed when the stalled operation is acknowledged late', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 100 })+ const stalled = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(await stalled).toBe(failures[0])++ worker.emit('message', {+ id: worker.sent[1].id,+ ok: true,+ result: 'late',+ } satisfies WorkerReply)+ await sleep(1)+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(failures[0].message)+ await expect(q.enqueueSync({ kind: 'ping' })).rejects.toBe(failures[0])+ expect(failures).toHaveLength(1)+ })++ it('waits for a shutdown that takes longer than syncTimeoutMs while within the bound', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, syncTimeoutMs: 30 })+ const closing = q.close()+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))+ await sleep(90)- ack(worker, worker.sent.find(m => m.op.kind === 'shutdown')!)++ ack(++ worker,++ worker.sent.find(m => m.op.kind === 'shutdown')!++ )+ // Liveness witness: close() resolves after the late shutdown ACK.+ await expect(closing).resolves.toBeUndefined()+ expect(failures).toHaveLength(0)+ })++ it('keeps the bound after an earlier failure, so close() does not wait forever', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })+ q.enqueueAsync('a', { kind: 'ping' })+ await vi.waitFor(() => expect(worker.sent).toHaveLength(2))+ const hung = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)+ expect(worker.sent).toHaveLength(3)+ worker.emit('message', {+ id: worker.sent[1].id,+ ok: false,+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },+ } satisfies WorkerReply)+ // Liveness witness: the async failure is latched first.+ await vi.waitFor(() => expect(failures).toHaveLength(1))+ const started = Date.now()+ await expect(q.close()).rejects.toThrow(/database or disk is full/)+ expect(Date.now() - started).toBeLessThan(400)+ expect(terminateCalls()).toBe(1)+ expect(await hung).toBeInstanceOf(Error)+ expect(failures).toHaveLength(1)+ })})diff --cc mcp-host/src/db/worker/dispatcher.tsindex 5c54620b4,d870380bb..a19d4680a--- a/mcp-host/src/db/worker/dispatcher.ts+++ b/mcp-host/src/db/worker/dispatcher.ts@@@ -428,8 -428,8 +428,7 @@@ export async function dispatch(op: Work// fire-and-forget lie this change removes.const tx = db.transaction((): ModelSelectionWriteOutcome => {const row = s.selectSessionModelSelectionRevision.get({ id: op.sessionId }) as-- | { model_selection_revision: number; model_selections: string | null }-- | undefined++ { model_selection_revision: number; model_selections: string | null } | undefinedif (!row) {const err: Error & { code?: string } = new Error(`update_session_model_selections: no session row for id=${op.sessionId}`@@@ -718,8 -718,8 +717,7 @@@}| undefinedconst pendingRow = s.selectPendingApprovalBySession.get(sessionRow.id) as-- | PendingApprovalRow-- | undefined++ PendingApprovalRow | undefinedreturn {session: sessionRow,messages: messageRows as PersistedSessionMessagePage['messages'],diff --cc mcp-host/src/guardrailHookResolver.tsindex 6a5e09807,9c7705214..968087ba3--- a/mcp-host/src/guardrailHookResolver.ts+++ b/mcp-host/src/guardrailHookResolver.ts@@@ -124,8 -147,8 +147,7 @@@ function endpointFor(cr: LlmHookCR, nsfunction isDuplicatePathLoser(cr: LlmHookCR): boolean {const conditions = (cr.status as-- | { conditions?: Array<{ type?: string; status?: string; reason?: string }> }-- | undefined++ { conditions?: Array<{ type?: string; status?: string; reason?: string }> } | undefined)?.conditionsreturn (Array.isArray(conditions) &&diff --cc monitoring/README.mdindex 48df12628,a17fe0a5a..2d2590851--- a/monitoring/README.md+++ b/monitoring/README.md@@@ -2,19 -2,15 +2,19 @@@Helm values and Grafana dashboard configs for the Clerum observability stack. This directory does not contain applications -- it provides configuration for Grafana and Loki deployed via their official Helm charts.+This starter does not configure a Prometheus scraper or Alertmanager. Control API+password-admission metrics and suggested alert queries are documented in+[password admission availability and monitoring](../docs/operations/password-admission.md).+## Contents- | Path | Description |- |------|-------------|- | `namespace.yaml` | Creates the `monitoring` Kubernetes namespace |- | `grafana/values.yaml` | Grafana Helm values (Loki datasource, dashboard provider, persistence, security context) |- | `loki/values.yaml` | Loki-stack Helm values (Loki + Promtail, 100 Gi persistence) |- | `dashboards/loki-grafana-dashboard.json` | Pre-built Grafana dashboard for Loki log exploration |- | `Makefile` | Install/upgrade/uninstall targets for both charts |+ | Path | Description |+ | ---------------------------------------- | ---------------------------------------------------------------------------------------- |+ | `namespace.yaml` | Creates the `monitoring` Kubernetes namespace |+ | `grafana/values.yaml` | Grafana Helm values (Loki datasource, dashboard provider, persistence, security context) |+ | `loki/values.yaml` | Loki-stack Helm values (Loki + Promtail, 100 Gi persistence by default) |+ | `dashboards/loki-grafana-dashboard.json` | Pre-built Grafana dashboard for Loki log exploration |+ | `Makefile` | Install/upgrade/uninstall targets for both charts |## Prerequisites