Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, vi } from 'vitest'
import express from 'express'
import request from 'supertest'
import { rootLogger } from '../src/observability/logger.js'
import { createAdminSecretsRouter } from '../src/routes/admin/secrets.js'
import { MockGateway } from './mockGateway.js'
function writeSummary(body: unknown) {
const write = body as {
name: string
namespace?: string
data?: Record<string, string>
stringData?: Record<string, string>
}
return {
name: write.name,
namespace: write.namespace || 'default',
keys: [
...new Set([...Object.keys(write.data ?? {}), ...Object.keys(write.stringData ?? {})]),
].sort((a, b) => a.localeCompare(b)),
}
}
function createGateway() {
return {
listSecrets: vi.fn(async () => [
{
metadata: { name: 's1', namespace: 'ns1', labels: { 'clerum.io/host-secret': 'true' } },
keys: ['openai-api-key'],
},
{ metadata: { name: 's2', namespace: 'ns1', labels: {} } },
{ metadata: { name: 's3', namespace: 'ns1', labels: { 'clerum.io/host-secret': 'false' } } },
]),
getSecret: vi.fn(async (_name: string, _namespace?: string): Promise<unknown> => {
throw Object.assign(new Error('not found'), { statusCode: 404 })
}),
createSecret: vi.fn(async (body: unknown) => writeSummary(body)),
updateSecret: vi.fn(async (body: unknown) => writeSummary(body)),
deleteSecret: vi.fn(async (name: string, namespace?: string) => ({
deleted: true,
name,
namespace: namespace || 'default',
})),
listResource: vi.fn(async () => [] as unknown[]),
}
}
describe('routes/secrets', () => {
it('supports list/create/update/delete (positive flow)', async () => {
const gateway = createGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const filtered = await request(app).get('/admin/secrets').expect(200)
// Producer-backed contract consumed by control-ui listLlmHostSecrets/SecretsTable.
// Secret values and Kubernetes metadata must never cross this boundary.
expect(filtered.body).toEqual({
items: [{ name: 's1', keys: ['openai-api-key'] }],
})
const filteredNames = (filtered.body.items || []).map((item: { name?: string }) => item.name)
expect(filteredNames).not.toContain('s2')
expect(filteredNames).not.toContain('s3')
expect(
(filtered.body.items || []).every(
(item: { name?: string }) => typeof item.name === 'string' && item.name.length > 0
)
).toBe(true)
await request(app)
.post('/admin/secrets')
.send({ name: 'new-secret', namespace: 'ns1', stringData: { token: 'abc' } })
.expect(201)
await request(app)
.put('/admin/secrets')
.send({ name: 'new-secret', namespace: 'ns1', stringData: { token: 'def' } })
.expect(200)
await request(app).delete('/admin/secrets/new-secret').expect(200)
expect(gateway.createSecret).toHaveBeenCalled()
expect(gateway.updateSecret).toHaveBeenCalled()
expect(gateway.deleteSecret).toHaveBeenCalledWith('new-secret', 'mcp-host')
})
it('returns 500 when gateway fails (edge case)', async () => {
const gateway = createGateway()
gateway.createSecret.mockImplementation(async () => {
throw new Error('invalid secret payload')
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
app.use(
(err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
res.status(500).json({ error: err instanceof Error ? err.message : 'unknown' })
}
)
const res = await request(app).post('/admin/secrets').send({}).expect(500)
expect(res.body.error).toContain('invalid secret payload')
})
describe('recipe-secrets', () => {
const labelledSecrets: Record<
string,
{ labels: Record<string, string>; data: Record<string, string> }
> = {
r1: {
labels: { 'clerum.io/recipe-secret': 'true' },
// base64 of "old-api" and "postgres://old"
data: { API_KEY: 'b2xkLWFwaQ==', DB_URL: 'cG9zdGdyZXM6Ly9vbGQ=' },
},
'coord-token': { labels: {}, data: {} },
other: { labels: { 'clerum.io/recipe-secret': 'false' }, data: {} },
}
function createRecipeGateway() {
return {
listSecrets: vi.fn(async (namespace?: string) =>
namespace === 'sandbox-recipes'
? [
{
metadata: {
name: 'r1',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe-secret': 'true' },
},
keys: ['API_KEY', 'DB_URL'],
},
{
metadata: { name: 'coord-token', namespace: 'sandbox-recipes', labels: {} },
keys: [],
},
{
metadata: {
name: 'other',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe-secret': 'false' },
},
keys: [],
},
]
: []
),
getSecret: vi.fn(async (name: string) => {
// Mirror the real apiserver: a missing Secret is a 404-shaped
// rejection. A bare Error here would only pass while the route
// collapsed EVERY read failure into "not found".
if (!(name in labelledSecrets)) {
throw Object.assign(new Error('not found'), { statusCode: 404 })
}
const entry = labelledSecrets[name]
return {
metadata: {
name,
namespace: 'sandbox-recipes',
uid: `uid-${name}`,
resourceVersion: '1',
labels: entry.labels,
},
data: entry.data,
}
}),
createSecret: vi.fn(async (body: unknown) => {
const summary = writeSummary(body)
return { ...summary, uid: `uid-${summary.name}`, resourceVersion: '1' }
}),
updateSecret: vi.fn(async (body: unknown) => writeSummary(body)),
deleteSecret: vi.fn(async (name: string, namespace?: string) => ({
deleted: true,
name,
namespace: namespace || 'default',
})),
}
}
it('lists only label-tagged recipe secrets with keys', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).get('/admin/recipe-secrets').expect(200)
expect(res.body.items).toHaveLength(1)
// r1's fixture has the recipe-secret label but no ownership label — the
// route classifies that as `unlabeled` (the degenerate case WRC refuses
// to project until an operator adds shared/owner-recipe).
expect(res.body.items[0]).toEqual({
name: 'r1',
namespace: 'sandbox-recipes',
keys: ['API_KEY', 'DB_URL'],
ownership: { kind: 'unlabeled' },
})
})
it('keeps recipe secret listing available when an auxiliary runtime namespace is unavailable', async () => {
const gateway = createRecipeGateway()
gateway.listSecrets.mockImplementation(async (namespace?: string) => {
if (namespace === 'sandbox-ui') {
const err = new Error('forbidden') as Error & { statusCode?: number }
err.statusCode = 403
throw err
}
return namespace === 'sandbox-recipes'
? [
{
metadata: {
name: 'r1',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe-secret': 'true' },
},
keys: ['API_KEY', 'DB_URL'],
},
]
: []
})
const warnSpy = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).get('/admin/recipe-secrets').expect(200)
expect(res.body.items).toHaveLength(1)
expect(res.body.items[0]).toMatchObject({ name: 'r1', namespace: 'sandbox-recipes' })
expect(warnSpy).toHaveBeenCalledWith(
expect.objectContaining({
event: 'recipe-secret-namespace-list-degraded',
namespace: 'sandbox-ui',
}),
'Recipe Secret namespace listing degraded'
)
warnSpy.mockRestore()
})
it('fails fast when sandbox-recipes recipe secret listing is unavailable', async () => {
const gateway = createRecipeGateway()
gateway.listSecrets.mockImplementation(async (namespace?: string) => {
if (namespace === 'sandbox-recipes') {
const err = new Error('sandbox-recipes unavailable') as Error & { statusCode?: number }
err.statusCode = 500
throw err
}
return []
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
app.use(
(
err: unknown,
_req: express.Request,
res: express.Response,
_next: express.NextFunction
) => {
res.status(500).json({ error: err instanceof Error ? err.message : 'unknown' })
}
)
const res = await request(app).get('/admin/recipe-secrets').expect(500)
expect(res.body.error).toContain('sandbox-recipes unavailable')
})
it('creates a recipe secret in sandbox-recipes with the recipe-secret label', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.post('/admin/recipe-secrets')
.send({
name: 'my-recipe',
data: { TOKEN: 'abc' },
ownership: { kind: 'shared' },
})
.expect(201)
expect(res.body).toMatchObject({
name: 'my-recipe',
namespace: 'sandbox-recipes',
ownership: { kind: 'shared' },
created: true,
})
expect(gateway.createSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'my-recipe',
namespace: 'sandbox-recipes',
type: 'Opaque',
labels: { 'clerum.io/recipe-secret': 'true', 'clerum.io/shared': 'true' },
stringData: { TOKEN: 'abc' },
})
)
})
it('creates a recipe secret in an allowed runtime namespace', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.post('/admin/recipe-secrets')
.send({
name: 'transport-creds',
targetNamespace: 'mcp-server',
data: { TOKEN: 'abc' },
ownership: { kind: 'shared' },
})
.expect(201)
expect(res.body.namespace).toBe('mcp-server')
expect(gateway.createSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'transport-creds',
namespace: 'mcp-server',
labels: { 'clerum.io/recipe-secret': 'true', 'clerum.io/shared': 'true' },
})
)
})
it('rejects recipe secret creation outside the workflow secret namespace allowlist', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.post('/admin/recipe-secrets')
.send({
name: 'bad-target',
targetNamespace: 'kube-system',
data: { TOKEN: 'abc' },
ownership: { kind: 'shared' },
})
.expect(400)
expect(res.body.error).toContain('targetNamespace must be one of')
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('rejects invalid recipe secret name', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.post('/admin/recipe-secrets')
.send({ name: 'BadName!', data: { K: 'v' } })
.expect(400)
expect(res.body.error).toContain('Invalid secret name')
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('rejects recipe secret with empty data', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.post('/admin/recipe-secrets')
.send({ name: 'ok-name', data: {} })
.expect(400)
expect(res.body.error).toContain('data is required')
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('PUT merges new values with existing keys (single-key update preserves untouched keys)', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'r1', data: { API_KEY: 'new-api' } })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(call).toMatchObject({
name: 'r1',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe-secret': 'true' },
})
// API_KEY overwritten with base64("new-api"); DB_URL preserved unchanged.
expect(call.data.API_KEY).toBe(Buffer.from('new-api', 'utf8').toString('base64'))
expect(call.data.DB_URL).toBe('cG9zdGdyZXM6Ly9vbGQ=')
expect(call.stringData).toBeUndefined()
})
it('PUT updates a recipe secret in an allowed runtime namespace', async () => {
const gateway = createRecipeGateway()
gateway.getSecret.mockImplementation(async (name: string, namespace?: string) => {
if (name === 'ui-creds' && namespace === 'sandbox-ui') {
return {
metadata: {
name,
namespace,
uid: `uid-${name}`,
resourceVersion: '1',
labels: { 'clerum.io/recipe-secret': 'true', 'clerum.io/shared': 'true' },
},
data: {},
}
}
throw new Error('not found')
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'ui-creds', targetNamespace: 'sandbox-ui', data: { K: 'new-api' } })
.expect(200)
expect(gateway.updateSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'ui-creds',
namespace: 'sandbox-ui',
labels: { 'clerum.io/recipe-secret': 'true', 'clerum.io/shared': 'true' },
}),
{ uid: 'uid-ui-creds', resourceVersion: '1' }
)
})
it('rejects recipe secret update outside the workflow secret namespace allowlist', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'r1', targetNamespace: 'kube-system', data: { K: 'new-api' } })
.expect(400)
expect(res.body.error).toContain('targetNamespace must be one of')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('PUT drops keys listed in removeKeys', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'r1', removeKeys: ['DB_URL'] })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(Object.keys(call.data)).toEqual(['API_KEY'])
expect(call.data.API_KEY).toBe('b2xkLWFwaQ==')
})
it('PUT rejects an update that would leave the secret with zero keys', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'r1', removeKeys: ['API_KEY', 'DB_URL'] })
.expect(400)
expect(res.body.error).toContain('at least one key')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('PUT rejects an empty payload (no data, no removeKeys)', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).put('/admin/recipe-secrets').send({ name: 'r1' }).expect(400)
expect(res.body.error).toContain('data or removeKeys')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('deletes a recipe secret from sandbox-recipes', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.delete('/admin/recipe-secrets/r1')
.send({ uid: 'uid-r1', resourceVersion: '1' })
.expect(200)
expect(gateway.deleteSecret).toHaveBeenCalledWith('r1', 'sandbox-recipes', {
uid: 'uid-r1',
resourceVersion: '1',
})
})
it('keeps recipe Secret DELETE identity-gated for clients without a body', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).delete('/admin/recipe-secrets/r1').expect(428)
expect(res.body).toMatchObject({ error: 'secret_identity_precondition_required' })
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
// The request above is exactly what a browser tab still running a UI bundle
// from before this contract sends, and that bundle cannot map the code: its
// error formatter reads `message` first and otherwise renders the raw body.
// Without a message the operator is shown
// "428 Precondition Required - secret_identity_precondition_required" and is
// told nothing about the reload that fixes it. The remedy therefore has to
// travel in the response, not in the client that is too old to hold it.
it('tells a client too old to map the code how to recover from the 428', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).delete('/admin/recipe-secrets/r1').expect(428)
expect(res.body.error).toBe('secret_identity_precondition_required')
expect(typeof res.body.message).toBe('string')
expect(res.body.message).toMatch(/reload the page/i)
// The remedy must not be the only thing this exit produced: the refusal is
// still fail-closed and the Secret is untouched.
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
// The 428 above only proves a body is REQUIRED; nothing proved the body is
// COMPARED. Replacing the comparison with `false` — deleting whatever the
// caller named, whoever replaced it in the meantime — left the whole suite
// green. The witness in each case is the live read: it proves the route
// reached the fence rather than bailing out earlier.
it.each([
{ label: 'uid', body: { uid: 'uid-someone-elses', resourceVersion: '1' } },
{ label: 'resourceVersion', body: { uid: 'uid-r1', resourceVersion: '2' } },
])(
'refuses a recipe Secret DELETE whose $label no longer matches the live object',
async testCase => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.delete('/admin/recipe-secrets/r1')
.send(testCase.body)
.expect(409)
expect(res.body).toMatchObject({
error: 'secret_identity_changed',
outcome: 'repair_required',
})
expect(gateway.getSecret).toHaveBeenCalled()
expect(gateway.deleteSecret).not.toHaveBeenCalled()
}
)
it('refuses a recipe Secret DELETE when the live object carries no identity', async () => {
const gateway = createRecipeGateway()
// A Secret that exists but whose identity the apiserver did not return:
// deleting it would be a name-addressed delete with no fence at all.
gateway.getSecret.mockResolvedValueOnce({
metadata: {
name: 'r1',
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe-secret': 'true' },
},
data: {},
} as never)
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.delete('/admin/recipe-secrets/r1')
.send({ uid: 'uid-r1', resourceVersion: '1' })
.expect(503)
expect(res.body).toMatchObject({
error: 'secret_identity_unavailable',
outcome: 'repair_required',
})
expect(gateway.getSecret).toHaveBeenCalled()
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
it('deletes a recipe secret from an allowed runtime namespace', async () => {
const gateway = createRecipeGateway()
gateway.getSecret.mockImplementation(async (name: string, namespace?: string) => {
if (name === 'ui-creds' && namespace === 'sandbox-ui') {
return {
metadata: {
name,
namespace,
uid: `uid-${name}`,
resourceVersion: '1',
labels: { 'clerum.io/recipe-secret': 'true', 'clerum.io/shared': 'true' },
},
data: {},
}
}
throw new Error('not found')
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.delete('/admin/recipe-secrets/ui-creds?targetNamespace=sandbox-ui')
.send({ uid: 'uid-ui-creds', resourceVersion: '1' })
.expect(200)
expect(gateway.deleteSecret).toHaveBeenCalledWith('ui-creds', 'sandbox-ui', {
uid: 'uid-ui-creds',
resourceVersion: '1',
})
})
it('rejects recipe secret delete outside the workflow secret namespace allowlist', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.delete('/admin/recipe-secrets/r1?targetNamespace=kube-system')
.expect(400)
expect(res.body.error).toContain('targetNamespace must be one of')
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
it('refuses to PUT a sandbox-recipes Secret that lacks the recipe-secret label', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'coord-token', data: { TOKEN: 'stolen' } })
.expect(404)
expect(res.body.error).toContain('Recipe secret not found')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('refuses to PUT a secret with the wrong label value', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'other', data: { TOKEN: 'x' } })
.expect(404)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it("refuses to PUT a secret that doesn't exist", async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'ghost', data: { TOKEN: 'x' } })
.expect(404)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('refuses to DELETE a sandbox-recipes Secret that lacks the recipe-secret label', async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app).delete('/admin/recipe-secrets/coord-token').expect(404)
expect(res.body.error).toContain('Recipe secret not found')
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
it("refuses to DELETE a secret that doesn't exist", async () => {
const gateway = createRecipeGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
await request(app).delete('/admin/recipe-secrets/ghost').expect(404)
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
// Only a 404 may become "Recipe secret not found". Any other read failure
// must propagate: the identity fence below cannot compare against a live
// object it never saw, and a 404 here would tell the caller the Secret is
// gone while it still exists on the cluster. The propagated error keeps
// its own status: a 4xx from the apiserver is forwarded as that 4xx (the
// Express default handler and `clerumErrorHandler` both read
// `.statusCode`), while a 5xx or status-less failure surfaces as 500.
it.each([
{
label: 'forbidden',
error: Object.assign(new Error('forbidden'), { statusCode: 403 }),
expectedStatus: 403,
},
{
label: 'apiserver outage',
error: Object.assign(new Error('boom'), { code: 500 }),
expectedStatus: 500,
},
{
label: 'status-less transport error',
error: new Error('socket hang up'),
expectedStatus: 500,
},
])('DELETE fails loud when the live read fails with a $label', async testCase => {
const gateway = createRecipeGateway()
gateway.getSecret.mockRejectedValueOnce(testCase.error)
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.delete('/admin/recipe-secrets/r1')
.send({ uid: 'uid-r1', resourceVersion: '1' })
.expect(testCase.expectedStatus)
expect(res.body).not.toMatchObject({ error: 'Recipe secret not found' })
// Liveness witness: the route reached the read and stopped there.
expect(gateway.getSecret).toHaveBeenCalledOnce()
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
it.each([
{
label: 'forbidden',
error: Object.assign(new Error('forbidden'), { statusCode: 403 }),
expectedStatus: 403,
},
{
label: 'apiserver outage',
error: Object.assign(new Error('boom'), { code: 500 }),
expectedStatus: 500,
},
])('PUT fails loud when the live read fails with a $label', async testCase => {
const gateway = createRecipeGateway()
gateway.getSecret.mockRejectedValueOnce(testCase.error)
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const res = await request(app)
.put('/admin/recipe-secrets')
.send({ name: 'r1', data: { API_KEY: 'rotated' } })
.expect(testCase.expectedStatus)
expect(res.body).not.toMatchObject({ error: 'Recipe secret not found' })
expect(gateway.getSecret).toHaveBeenCalledOnce()
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
})
it('silently ignores ?namespace= query parameter and uses config namespace', async () => {
const gateway = createGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
// Namespace in query string is silently ignored — requests succeed (200, not 400)
await request(app).get('/admin/secrets?namespace=evil').expect(200)
await request(app).delete('/admin/secrets/s1?namespace=evil').expect(200)
// Gateway WAS called despite namespace in query (it's silently ignored)
expect(gateway.listSecrets).toHaveBeenCalled()
expect(gateway.deleteSecret).toHaveBeenCalled()
// Security audit was logged
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('"alert":"SECURITY"'))
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('"vector":"query-param"'))
warnSpy.mockRestore()
})
})
// Slot-aware LLM credential validation (spec R4.5.3 / §6 control-api row).
describe('routes/secrets — LLM slot-aware validation', () => {
function makeApp() {
const gateway = createGateway()
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
return { app, gateway }
}
const validVertexJson = JSON.stringify({
type: 'service_account',
client_email: '[email protected]',
private_key: '-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n',
})
it('rejects a partial Bedrock pair (only access-key-id) with 400', async () => {
const { app, gateway } = makeApp()
const res = await request(app)
.post('/admin/secrets')
.send({ name: 'chatllm-api-keys', stringData: { 'aws-access-key-id': 'AKIA...' } })
.expect(400)
expect(res.body.error).toMatch(/Bedrock/i)
expect(res.body.error).toMatch(/aws-secret-access-key/)
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('rejects a partial Bedrock pair on update (PUT) with 400', async () => {
const { app, gateway } = makeApp()
await request(app)
.put('/admin/secrets')
.send({ name: 'chatllm-api-keys', stringData: { 'aws-secret-access-key': 'secret' } })
.expect(400)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('accepts the complete Bedrock pair written together', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({
name: 'chatllm-api-keys',
stringData: { 'aws-access-key-id': 'AKIA...', 'aws-secret-access-key': 'secret' },
})
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
it('rejects a malformed Vertex service-account JSON with 400', async () => {
const { app, gateway } = makeApp()
const res = await request(app)
.post('/admin/secrets')
.send({ name: 'chatllm-api-keys', stringData: { 'vertex-service-account-json': 'not-json' } })
.expect(400)
expect(res.body.error).toMatch(/vertex-service-account-json/)
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('rejects a Vertex JSON missing client_email / private_key with 400', async () => {
const { app, gateway } = makeApp()
const res = await request(app)
.post('/admin/secrets')
.send({
name: 'chatllm-api-keys',
stringData: { 'vertex-service-account-json': JSON.stringify({ type: 'service_account' }) },
})
.expect(400)
expect(res.body.error).toMatch(/client_email/)
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('accepts a well-formed Vertex service-account JSON', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({
name: 'chatllm-api-keys',
stringData: { 'vertex-service-account-json': validVertexJson },
})
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
it('also validates base64 `data` payloads (Vertex JSON)', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({
name: 'chatllm-api-keys',
data: { 'vertex-service-account-json': Buffer.from('not-json').toString('base64') },
})
.expect(400)
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('leaves the generic (non-LLM) secret contract untouched', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({ name: 'generic', stringData: { token: 'abc', 'some-key': 'v' } })
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
it('accepts a single-key provider (openai) unchanged', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({ name: 'chatllm-api-keys', stringData: { 'openai-api-key': 'sk-...' } })
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
// FIX 3: the slot-aware rule is scoped to the known LLM Secret name(s). A
// generic Secret carrying a lone `aws-access-key-id` (e.g. a plain AWS
// credential Secret) must NOT trip the bedrock paired-slot rule.
it('does not apply the bedrock pair rule to a non-LLM secret with a lone aws-access-key-id', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({ name: 'my-aws-creds', stringData: { 'aws-access-key-id': 'AKIA...' } })
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
// Topic 1b: the gate now also fires on the host-secret LABEL, so a per-host
// NAMED Secret (not `chatllm-api-keys`) is validated too — it can no longer
// bypass the bedrock/vertex contract just by having a different name.
it('validates a per-host-NAMED labeled secret: half Bedrock pair → 400', async () => {
const { app, gateway } = makeApp()
const res = await request(app)
.post('/admin/secrets')
.send({
name: 'host-abc-llm',
labels: { 'clerum.io/host-secret': 'true' },
stringData: { 'aws-access-key-id': 'AKIA...' },
})
.expect(400)
expect(res.body.error).toMatch(/Bedrock/i)
expect(res.body.error).toMatch(/aws-secret-access-key/)
expect(gateway.createSecret).not.toHaveBeenCalled()
})
it('accepts a valid per-host labeled secret (complete Bedrock pair)', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({
name: 'host-abc-llm',
labels: { 'clerum.io/host-secret': 'true' },
stringData: { 'aws-access-key-id': 'AKIA...', 'aws-secret-access-key': 'secret' },
})
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
// The label is what arms the gate — a per-host NAME without the label is still
// a generic Secret (no slot contract), matching the FIX-3 scoping above.
it('does NOT validate a per-host-named secret that lacks the host-secret label', async () => {
const { app, gateway } = makeApp()
await request(app)
.post('/admin/secrets')
.send({ name: 'host-abc-llm', stringData: { 'aws-access-key-id': 'AKIA...' } })
.expect(201)
expect(gateway.createSecret).toHaveBeenCalled()
})
})
// FIX 2b: opt-in server-side merge (read-then-replace) for PUT /admin/secrets.
describe('routes/secrets — PUT merge semantics', () => {
function makeMergeApp(
existing: { data?: Record<string, string>; labels?: Record<string, string> } | null
) {
const gateway = createGateway()
gateway.getSecret.mockImplementation(async () => {
if (existing === null) throw new Error('not found')
return {
metadata: {
name: 'chatllm-api-keys',
namespace: 'mcp-host',
labels: existing.labels || {},
},
data: existing.data || {},
}
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
return { app, gateway }
}
const b64 = (s: string): string => Buffer.from(s, 'utf8').toString('base64')
it('merge:true preserves keys the caller did not send (other providers survive)', async () => {
const { app, gateway } = makeMergeApp({
data: { 'openai-api-key': b64('sk-existing'), 'claude-api-key': b64('claude-existing') },
})
const res = await request(app)
.put('/admin/secrets')
.send({ name: 'chatllm-api-keys', merge: true, stringData: { 'openai-api-key': 'sk-new' } })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
// openai overwritten, claude preserved untouched.
expect(call.data['openai-api-key']).toBe(b64('sk-new'))
expect(call.data['claude-api-key']).toBe(b64('claude-existing'))
expect(call.stringData).toBeUndefined()
// The response must NEVER echo secret values (names-only R4 contract) —
// especially not the preserved values of keys the caller did not send.
expect(res.body).toEqual({
name: 'chatllm-api-keys',
namespace: 'mcp-host',
keys: ['claude-api-key', 'openai-api-key'],
})
const serialized = JSON.stringify(res.body)
expect(serialized).not.toContain(b64('claude-existing'))
expect(serialized).not.toContain(b64('sk-new'))
})
it('merge:true validates the EFFECTIVE merge (bedrock pair completed across existing+sent)', async () => {
const { app, gateway } = makeMergeApp({
data: { 'aws-access-key-id': b64('AKIA-existing') },
})
await request(app)
.put('/admin/secrets')
.send({
name: 'chatllm-api-keys',
merge: true,
stringData: { 'aws-secret-access-key': 'secret-new' },
})
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(call.data['aws-access-key-id']).toBe(b64('AKIA-existing'))
expect(call.data['aws-secret-access-key']).toBe(b64('secret-new'))
})
it('merge:true still 400s when the pair is incomplete AFTER the merge', async () => {
const { app, gateway } = makeMergeApp({ data: { 'openai-api-key': b64('sk') } })
const res = await request(app)
.put('/admin/secrets')
.send({
name: 'chatllm-api-keys',
merge: true,
stringData: { 'aws-access-key-id': 'AKIA...' },
})
.expect(400)
expect(res.body.error).toMatch(/Bedrock/i)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('merge:true skips empty values (blanking is not deletion)', async () => {
const { app, gateway } = makeMergeApp({
data: { 'openai-api-key': b64('sk-existing'), 'claude-api-key': b64('claude-existing') },
})
await request(app)
.put('/admin/secrets')
.send({
name: 'chatllm-api-keys',
merge: true,
stringData: { 'openai-api-key': '', 'claude-api-key': 'claude-new' },
})
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
// The empty value did not wipe the stored openai key; claude was updated.
expect(call.data['openai-api-key']).toBe(b64('sk-existing'))
expect(call.data['claude-api-key']).toBe(b64('claude-new'))
})
it('merge:true on a missing secret returns 404', async () => {
const { app, gateway } = makeMergeApp(null)
await request(app)
.put('/admin/secrets')
.send({ name: 'chatllm-api-keys', merge: true, stringData: { 'openai-api-key': 'sk' } })
.expect(404)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('without the flag the update stays full-replace (data passed through, not merged)', async () => {
const { app, gateway } = makeMergeApp({ data: { 'openai-api-key': b64('sk-existing') } })
await request(app)
.put('/admin/secrets')
.send({ name: 'chatllm-api-keys', stringData: { 'claude-api-key': 'claude-new' } })
.expect(200)
// Full-replace passes the body through verbatim — it does NOT merge the
// existing data. (It may do a validation-only label read to arm the slot
// gate, but the write body carries only the caller's keys.)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(call.stringData).toEqual({ 'claude-api-key': 'claude-new' })
expect(call.data).toBeUndefined()
})
// Closes the full-replace bypass: a per-host labeled LLM Secret written via
// full-replace WITHOUT echoing labels must still be slot-validated, because
// updateSecret re-attaches the existing host-secret label. Identity is armed
// from the EXISTING labels, not the (omitted) body labels.
it('full-replace of a labeled per-host secret is slot-validated even when body omits labels', async () => {
const { app, gateway } = makeMergeApp({
data: { 'openai-api-key': b64('sk-existing') },
labels: { 'clerum.io/host-secret': 'true' },
})
const res = await request(app)
.put('/admin/secrets')
// per-host NAME (not chatllm-api-keys), no labels in body, half Bedrock pair
.send({ name: 'host-x-llm', stringData: { 'aws-access-key-id': 'AKIA...' } })
.expect(400)
expect(res.body.error).toMatch(/Bedrock/i)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
// Topic 1b Task 2 — removeKeys: the delete-a-slot half of partial edit, so an
// operator retiring a provider from the fallback chain can drop that slot.
it('merge:true removeKeys retires a named slot, preserving the rest (label-armed host secret)', async () => {
const { app, gateway } = makeMergeApp({
data: { 'openai-api-key': b64('sk'), 'claude-api-key': b64('cl') },
labels: { 'clerum.io/host-secret': 'true' },
})
const res = await request(app)
.put('/admin/secrets')
// A per-host NAME (not chatllm-api-keys): identity comes from the label.
.send({ name: 'host-x-llm', merge: true, removeKeys: ['openai-api-key'] })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(Object.keys(call.data)).toEqual(['claude-api-key'])
expect(call.data['claude-api-key']).toBe(b64('cl'))
// Names-only response — never echoes values.
expect(res.body.keys).toEqual(['claude-api-key'])
expect(JSON.stringify(res.body)).not.toContain(b64('cl'))
})
it('merge:true removeKeys touches only the named keys (does not wipe others)', async () => {
const { app, gateway } = makeMergeApp({
data: {
'openai-api-key': b64('sk'),
'claude-api-key': b64('cl'),
'gemini-api-key': b64('g'),
},
labels: { 'clerum.io/host-secret': 'true' },
})
await request(app)
.put('/admin/secrets')
.send({ name: 'host-x-llm', merge: true, removeKeys: ['gemini-api-key'] })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(Object.keys(call.data).sort()).toEqual(['claude-api-key', 'openai-api-key'])
})
// Contract: the Bedrock pair is atomic on the way OUT too — removing one half
// retires the whole pair so the Secret never lands in a half-pair state.
it('merge:true removeKeys of one Bedrock key retires the whole pair', async () => {
const { app, gateway } = makeMergeApp({
data: {
'aws-access-key-id': b64('AKIA'),
'aws-secret-access-key': b64('sec'),
'openai-api-key': b64('sk'),
},
labels: { 'clerum.io/host-secret': 'true' },
})
await request(app)
.put('/admin/secrets')
.send({ name: 'host-x-llm', merge: true, removeKeys: ['aws-access-key-id'] })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
// Both Bedrock keys gone; openai untouched.
expect(Object.keys(call.data)).toEqual(['openai-api-key'])
})
it('merge:true removeKeys that would empty the secret → 400 (must retain one key)', async () => {
const { app, gateway } = makeMergeApp({
data: { 'openai-api-key': b64('sk') },
labels: { 'clerum.io/host-secret': 'true' },
})
const res = await request(app)
.put('/admin/secrets')
.send({ name: 'host-x-llm', merge: true, removeKeys: ['openai-api-key'] })
.expect(400)
expect(res.body.error).toMatch(/at least one key/)
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('merge:true removeKeys works on a generic (non-LLM) secret without slot logic', async () => {
const { app, gateway } = makeMergeApp({ data: { a: b64('1'), b: b64('2') } })
await request(app)
.put('/admin/secrets')
.send({ name: 'generic', merge: true, removeKeys: ['a'] })
.expect(200)
const call = (gateway.updateSecret as any).mock.calls[0][0]
expect(Object.keys(call.data)).toEqual(['b'])
})
})
describe('routes/secrets — fallback credential protection', () => {
const secretName = 'shared-llm'
const fallbackSlot = 'claude-api-key-fb1'
const b64 = (value: string): string => Buffer.from(value, 'utf8').toString('base64')
async function makeFallbackProtectedApp() {
const gateway = new MockGateway('mcp-host')
await gateway.createResource(
'hosts',
{
metadata: { name: 'agent-a' },
spec: {
secretRef: secretName,
model: { provider: 'openai', name: 'gpt-5.4-mini' },
llmPolicy: {
fallbacks: [
{
provider: 'claude',
model: 'claude-sonnet-4-6',
credentialSlot: fallbackSlot,
},
],
},
},
},
'mcp-host'
)
gateway.seedSecret(secretName, 'mcp-host', {
labels: { 'clerum.io/host-secret': 'true' },
data: {
'openai-api-key': b64('openai-secret'),
[fallbackSlot]: b64('claude-secret'),
},
})
const app = express()
app.use(express.json())
app.use(createAdminSecretsRouter(gateway as never))
return { app, gateway }
}
it('blocks merge removeKeys for a slot used by a producer-created Host fallback', async () => {
const { app, gateway } = await makeFallbackProtectedApp()
vi.spyOn(gateway, 'updateSecret')
const res = await request(app)
.put('/admin/secrets')
.send({ name: secretName, merge: true, removeKeys: [fallbackSlot] })
.expect(409)
expect(res.body.error).toContain(fallbackSlot)
expect(res.body.error).toContain('active fallback')
expect(res.body.error).toContain('agent-a')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('blocks full replacement when it omits a slot used by a producer-created Host fallback', async () => {
const { app, gateway } = await makeFallbackProtectedApp()
vi.spyOn(gateway, 'updateSecret')
const res = await request(app)
.put('/admin/secrets')
.send({ name: secretName, stringData: { 'openai-api-key': 'replacement' } })
.expect(409)
expect(res.body.error).toContain(fallbackSlot)
expect(res.body.error).toContain('active fallback')
expect(res.body.error).toContain('agent-a')
expect(gateway.updateSecret).not.toHaveBeenCalled()
})
it('blocks deleting a Secret referenced by a producer-created Host fallback', async () => {
const { app, gateway } = await makeFallbackProtectedApp()
vi.spyOn(gateway, 'deleteSecret')
const res = await request(app).delete(`/admin/secrets/${secretName}`).expect(409)
expect(res.body.error).toContain(fallbackSlot)
expect(res.body.error).toContain('active fallback')
expect(res.body.error).toContain('agent-a')
expect(gateway.deleteSecret).not.toHaveBeenCalled()
})
})