Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/** External model boundary for the isolated approved-tools E2E image only. */
import { createHash, randomUUID } from 'node:crypto'
import { isDeepStrictEqual } from 'node:util'
type Row = Record<string, unknown>
const COMPLETIONS = 'https://chatgpt.com/backend-api/codex/responses'
const CATALOG = 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0'
const BRIDGES = ['clerum__tool_search', 'clerum__tool_describe', 'clerum__tool_call']
// The deterministic external model is scoped to this fixture's read journey.
const FIXTURE_QUERY = 'verification receipt'
// A user turn with this text gets one response carrying more tool calls than
// the Codex contract allows (maxToolCalls 256), so the proxy must reject it.
const LIMIT_PROBE = 'tool call limit probe'
const LIMIT_PROBE_CALLS = 257
// A user turn with this text gets exactly maxToolCalls distinct search calls;
// the continuation must carry every result back before the final answer.
const LIMIT_BOUNDARY = 'tool call limit boundary'
const LIMIT_BOUNDARY_CALLS = 256
function record(value: unknown): Row {
if (!value || typeof value !== 'object' || Array.isArray(value))
throw new Error('expected_object')
return value as Row
}
function parse(value: unknown): Row {
if (typeof value !== 'string') throw new Error('expected_json_text')
return record(JSON.parse(value))
}
function textResult(value: unknown, expectedName: string): Row {
if (typeof value === 'string' && value.startsWith('<tool_output')) {
const wrapped =
/^<tool_output name="([^"]+)" sanitized="(?:true|false)">\n([\s\S]*)\n<\/tool_output>$/.exec(
value
)
if (!wrapped || wrapped[1] !== expectedName || wrapped[2]!.includes('</tool_output>')) {
throw new Error('invalid_tool_output_envelope')
}
value = wrapped[2]
}
// Real tool failures can be plain sanitized text; preserve that failure as
// data rather than inventing a successful result or retrying an approval.
if (typeof value === 'string' && !value.trim().startsWith('{')) return { error: value }
const result = parse(value)
if (result.isError === true || result.is_error === true) return result
if (Array.isArray(result.content)) {
const blocks = result.content.map(record)
if (blocks.length !== 1 || blocks[0]?.type !== 'text')
throw new Error('expected_single_text_result')
return parse(blocks[0].text)
}
return result
}
type Exchange = { call: Row; args: Row; result: Row }
type Decision = { stage: string; args?: Row; result?: Row }
function expectedCall(exchange: Exchange, decision: Decision): void {
if (exchange.call.name !== decision.stage || !isDeepStrictEqual(exchange.args, decision.args)) {
throw new Error('unexpected_fixture_call')
}
}
function exchanges(rows: Row[]): Exchange[] {
const calls = rows.filter(row => row.type === 'function_call')
const outputs = rows.filter(row => row.type === 'function_call_output')
if (calls.length > 12 || calls.length !== outputs.length)
throw new Error('incomplete_or_repeated_calls')
if (new Set(calls.map(call => call.call_id)).size !== calls.length)
throw new Error('duplicate_call_id')
return calls.map(call => {
if (typeof call.call_id !== 'string' || typeof call.name !== 'string')
throw new Error('invalid_call')
const matched = outputs.filter(output => output.call_id === call.call_id)
if (matched.length !== 1 || rows.indexOf(matched[0]!) < rows.indexOf(call))
throw new Error('uncorrelated_result')
const args = parse(call.arguments)
const expectedName = call.name === BRIDGES[2] ? args.name : call.name
if (typeof expectedName !== 'string') throw new Error('invalid_envelope_target')
return { call, args, result: textResult(matched[0]!.output, expectedName) }
})
}
function failed(result: Row): boolean {
return result.isError === true || result.is_error === true || Boolean(result.error)
}
// Every call sent for the boundary turn must come back once, with a real
// search result after it. Returns the number of correlated results.
function boundaryResults(rows: Row[], sent: Set<string>): number {
const calls = rows.filter(row => row.type === 'function_call')
const outputs = rows.filter(row => row.type === 'function_call_output')
if (calls.length !== sent.size || outputs.length !== sent.size)
throw new Error('incomplete_boundary_batch')
if (new Set(calls.map(call => call.call_id)).size !== calls.length)
throw new Error('duplicate_call_id')
for (const call of calls) {
if (call.name !== BRIDGES[0] || typeof call.call_id !== 'string' || !sent.has(call.call_id))
throw new Error('uncorrelated_boundary_call')
const matched = outputs.filter(output => output.call_id === call.call_id)
if (matched.length !== 1 || rows.indexOf(matched[0]!) < rows.indexOf(call))
throw new Error('uncorrelated_result')
const result = textResult(matched[0]!.output, BRIDGES[0])
if (failed(result) || !Array.isArray(result.results)) throw new Error('boundary_search_failed')
}
return calls.length
}
function described(exchange: Exchange): string {
if (
exchange.call.name !== BRIDGES[1] ||
exchange.result.found !== true ||
exchange.result.name !== exchange.args.name ||
typeof exchange.args.name !== 'string'
)
throw new Error('description_target_mismatch')
const schema = record(exchange.result.parameters)
if (schema.type !== 'object' || (Array.isArray(schema.required) && schema.required.length))
throw new Error('fixture_requires_empty_arguments')
return exchange.args.name
}
function receipt(exchange: Exchange, candidate: string): Row {
if (
exchange.call.name !== BRIDGES[2] ||
exchange.args.name !== candidate ||
Object.keys(record(exchange.args.arguments)).length
)
throw new Error('invocation_target_mismatch')
const value = exchange.result
if (
typeof value.businessId !== 'string' ||
!value.businessId ||
typeof value.runId !== 'string' ||
typeof value.callId !== 'string' ||
typeof value.tool !== 'string' ||
!candidate.endsWith(`__${value.tool}`)
)
throw new Error('missing_business_result')
return value
}
function reusableDescription(history: Row[]): string | undefined {
let previousUser = history.length - 1
while (previousUser >= 0 && history[previousUser]?.role !== 'user') previousUser--
if (previousUser < 0) return undefined
try {
// A canceled or denied previous task invalidates reuse; an ordinary new
// request must recover via discovery, not continue its interrupted call.
const previous = exchanges(history.slice(previousUser + 1))
const last = previous.at(-1)
if (
!last ||
last.call.name !== BRIDGES[2] ||
failed(last.result) ||
typeof last.args.name !== 'string'
)
return undefined
receipt(last, last.args.name)
const allCalls = history.filter(row => row.type === 'function_call' && row.name === BRIDGES[1])
for (const call of allCalls.reverse()) {
const result = history.find(
row => row.type === 'function_call_output' && row.call_id === call.call_id
)
if (!result) continue
const candidate = described(exchanges([call, result])[0]!)
if (candidate === last.args.name) return candidate
}
} catch {
// Historical partial results are expected after genuine user cancellation.
// They cannot supply a reusable schema, but never block the new task.
}
return undefined
}
function receiptDecision(current: Exchange[], known?: string): Decision {
const last = current.at(-1)
if (!last)
return known
? { stage: BRIDGES[2]!, args: { name: known, arguments: {} } }
: { stage: BRIDGES[0]!, args: { query: FIXTURE_QUERY, limit: 5 } }
expectedCall(last, receiptDecision(current.slice(0, -1), known))
if (failed(last.result)) {
// An obsolete cached target can be rediscovered. Approval denial/cancel
// remains terminal and must never cause an automatic business retry.
if (
last.call.name === BRIDGES[2] &&
/not found|not available|unavailable|not .*catalog/i.test(JSON.stringify(last.result)) &&
!/approval|denied|cancel|reject/i.test(JSON.stringify(last.result))
)
return { stage: BRIDGES[0]!, args: { query: FIXTURE_QUERY, limit: 5 } }
return { stage: 'denied', result: last.result }
}
if (last.call.name === BRIDGES[0]) {
if (last.args.query !== FIXTURE_QUERY || !Array.isArray(last.result.results))
throw new Error('invalid_search_result')
if (last.result.found === 0 && last.result.results.length === 0)
return { stage: 'denied', result: last.result }
if (last.result.results.length !== 1) throw new Error('search_must_find_one_target')
const match = record(last.result.results[0])
if (
typeof match.name !== 'string' ||
BRIDGES.includes(match.name) ||
'parameters' in match ||
'inputSchema' in match
)
throw new Error('invalid_search_target')
return { stage: BRIDGES[1]!, args: { name: match.name } }
}
if (last.call.name === BRIDGES[1]) {
if (last.result.found === false)
return { stage: BRIDGES[0]!, args: { query: FIXTURE_QUERY, limit: 5 } }
const candidate = described(last)
const search = current.at(-2)
if (
!search ||
search.call.name !== BRIDGES[0] ||
!Array.isArray(search.result.results) ||
record(search.result.results[0]).name !== candidate
)
throw new Error('description_without_search')
return { stage: BRIDGES[2]!, args: { name: candidate, arguments: {} } }
}
if (last.call.name === BRIDGES[2]) {
const prior = current.at(-2)
const candidate = prior?.call.name === BRIDGES[1] ? described(prior) : known
if (!candidate) throw new Error('invocation_without_schema')
return { stage: 'final', result: receipt(last, candidate) }
}
throw new Error('unsupported_receipt_transition')
}
function workflowDecision(current: Exchange[], prompt: string): Decision {
if (!current.length) return { stage: 'workflow_list', args: {} }
expectedCall(current.at(-1)!, workflowDecision(current.slice(0, -1), prompt))
const listed = current[0]!
if (
listed.call.name !== 'workflow_list' ||
failed(listed.result) ||
!Array.isArray(listed.result.items)
)
throw new Error('workflow_list_required')
const matches = listed.result.items
.map(record)
.filter(item => typeof item.name === 'string' && prompt.includes(item.name))
if (matches.length !== 1 || matches[0]!.requiresInput === true)
throw new Error('workflow_not_uniquely_resolved')
const selected = matches[0]!
const args: Row = { name: selected.name }
if (Array.isArray(selected.targets)) {
const targets = selected.targets.map(record)
const matching =
targets.length === 1
? targets
: targets.filter(
target => typeof target.label === 'string' && prompt.includes(target.label)
)
if (matching.length !== 1 || typeof matching[0]!.label !== 'string')
throw new Error('ambiguous_workflow_target')
args.targetLabel = matching[0]!.label
}
const last = current.at(-1)!
if (failed(last.result)) return { stage: 'denied', result: last.result }
if (current.length === 1) return { stage: 'workflow_trigger', args }
for (const item of current.slice(1)) {
if (item.args.name !== selected.name) throw new Error('workflow_target_mismatch')
}
if (last.call.name === 'workflow_trigger' && current.length === 2) {
if (last.result.workflowName !== selected.name) throw new Error('trigger_binding_missing')
return { stage: 'workflow_status', args }
}
if (last.call.name === 'workflow_status' && current.length === 3) {
if (last.result.name !== selected.name) throw new Error('status_binding_missing')
const latest = last.result.latestRun ? record(last.result.latestRun) : undefined
if (
['failed', 'cancelled', 'canceled'].includes(
String(latest?.phase ?? last.result.workflowPhase)
)
)
return { stage: 'denied', result: last.result }
// workflow_result performs the real bounded artifact-readiness polling.
return { stage: 'workflow_result', args }
}
if (last.call.name === 'workflow_result' && current.length === 4) {
if (
last.result.workflowName !== selected.name ||
last.result.artifactAvailable !== true ||
typeof record(last.result.result).businessId !== 'string'
)
throw new Error('workflow_artifact_missing')
return { stage: 'final', result: last.result }
}
throw new Error('unsupported_workflow_transition')
}
function stream(...events: Row[]): Response {
return new Response(
[...events, { type: 'response.completed' }]
.map(row => `data: ${JSON.stringify(row)}\n\n`)
.join(''),
{
headers: { 'content-type': 'text/event-stream' },
}
)
}
export function createApprovedToolsUpstream() {
const evidence = {
catalogRequests: 0,
rejected: 0,
completions: 0,
searchCalls: 0,
describeCalls: 0,
businessCalls: 0,
finalResponses: 0,
deniedResponses: 0,
// Probe turns answered with an over-limit batch, every completion served
// for them, and completions that repeated an already answered probe turn.
limitProbe: { turns: 0, completions: 0, unexpectedRetries: 0 },
// Boundary turns answered with exactly maxToolCalls calls, completions
// served for them, correlated results received and final answers sent.
limitBoundary: {
turns: 0,
completions: 0,
toolResults: 0,
finalResponses: 0,
unexpectedRetries: 0,
},
// Only bounded measurements, never request or result contents.
requests: [] as Array<{
definitionCount: number
explicitNonStrictCount: number
definitionBytes: number
inputBytes: number
connectorDefinitionCount: number
leakedSchema: boolean
stage: string
}>,
}
// Digests of the history up to each answered probe user turn. A retry of the
// same turn resends that history unchanged.
const answeredProbeTurns = new Set<string>()
// Call IDs sent for each boundary turn, and whether its final answer was sent.
const boundaryTurns = new Map<string, { sent: Set<string>; finalized: boolean }>()
const fetchFn: typeof fetch = async (input, init) => {
try {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url
if (url === CATALOG && (!init?.method || init.method === 'GET')) {
evidence.catalogRequests++
return Response.json({
models: [{ slug: 'gpt-5.3-codex', display_name: 'Codex isolated tool test' }],
})
}
if (url !== COMPLETIONS || init?.method !== 'POST') throw new Error('operation_denied')
const payload = parse(init.body)
if (payload.stream !== true || payload.store !== false)
throw new Error('invalid_stream_contract')
if (!Array.isArray(payload.input) || !Array.isArray(payload.tools))
throw new Error('missing_input_or_tools')
const history = payload.input.map(record)
const tools = payload.tools.map(record)
const names = new Set(tools.map(tool => tool.name))
if (!BRIDGES.every(name => names.has(name))) throw new Error('missing_discovery_bridge')
if (names.size !== tools.length || tools.some(tool => tool.type !== 'function'))
throw new Error('invalid_definitions')
let lastUser = history.length - 1
while (lastUser >= 0 && history[lastUser]?.role !== 'user') lastUser--
const user = history[lastUser]
if (!user || typeof user.content !== 'string' || !user.content.trim())
throw new Error('missing_user_request')
if (user.content.toLowerCase().includes(LIMIT_BOUNDARY)) {
if (evidence.requests.length >= 512) throw new Error('evidence_capacity_exceeded')
evidence.limitBoundary.completions++
const turn = createHash('sha256')
.update(JSON.stringify(history.slice(0, lastUser + 1)))
.digest('hex')
const known = boundaryTurns.get(turn)
const rows = history.slice(lastUser + 1)
const measured = (stage: string) => ({
definitionCount: tools.length,
explicitNonStrictCount: tools.filter(tool => tool.strict === false).length,
definitionBytes: Buffer.byteLength(JSON.stringify(tools)),
inputBytes: Buffer.byteLength(JSON.stringify(payload.input)),
connectorDefinitionCount: 0,
leakedSchema: false,
stage,
})
if (rows.length === 0) {
if (known) {
evidence.limitBoundary.unexpectedRetries++
throw new Error('unexpected_retry')
}
// Distinct arguments: identical consecutive calls trip the host's
// doom-loop guard, which would test that guard instead of the limit.
const calls = Array.from({ length: LIMIT_BOUNDARY_CALLS }, (_, index) => ({
type: 'function_call',
id: randomUUID(),
call_id: randomUUID(),
name: BRIDGES[0],
arguments: JSON.stringify({ query: `limit boundary ${index}`, limit: 1 }),
}))
boundaryTurns.set(turn, {
sent: new Set(calls.map(call => call.call_id)),
finalized: false,
})
evidence.limitBoundary.turns++
evidence.completions++
evidence.requests.push(measured('limit_boundary'))
return stream(...calls.map(item => ({ type: 'response.output_item.done', item })))
}
if (!known) throw new Error('unknown_boundary_turn')
if (known.finalized) {
evidence.limitBoundary.unexpectedRetries++
throw new Error('unexpected_retry')
}
const received = boundaryResults(rows, known.sent)
known.finalized = true
evidence.limitBoundary.toolResults += received
evidence.limitBoundary.finalResponses++
evidence.completions++
evidence.requests.push(measured('limit_boundary_final'))
return stream({
type: 'response.output_text.delta',
delta: `Tool call limit boundary complete: ${received} tool results received.`,
})
}
if (user.content.toLowerCase().includes(LIMIT_PROBE)) {
if (evidence.requests.length >= 512) throw new Error('evidence_capacity_exceeded')
evidence.limitProbe.completions++
const turn = createHash('sha256')
.update(JSON.stringify(history.slice(0, lastUser + 1)))
.digest('hex')
// The key covers history up to the probe message, so both a retry and
// a continuation (an over-limit batch never yields tool results, so a
// continuation means the limit was not enforced) of an answered turn
// land here.
if (answeredProbeTurns.has(turn)) {
evidence.limitProbe.unexpectedRetries++
throw new Error('unexpected_retry')
}
answeredProbeTurns.add(turn)
evidence.limitProbe.turns++
evidence.completions++
evidence.requests.push({
definitionCount: tools.length,
explicitNonStrictCount: tools.filter(tool => tool.strict === false).length,
definitionBytes: Buffer.byteLength(JSON.stringify(tools)),
inputBytes: Buffer.byteLength(JSON.stringify(payload.input)),
connectorDefinitionCount: 0,
leakedSchema: false,
stage: 'limit_probe',
})
return stream(
...Array.from({ length: LIMIT_PROBE_CALLS }, () => ({
type: 'response.output_item.done',
item: {
type: 'function_call',
id: randomUUID(),
call_id: randomUUID(),
name: BRIDGES[0],
arguments: JSON.stringify({ query: FIXTURE_QUERY, limit: 5 }),
},
}))
)
}
const isWorkflow = /\bworkflow\b/i.test(user.content)
if (!isWorkflow && !user.content.toLowerCase().includes(FIXTURE_QUERY))
throw new Error('unsupported_fixture_task')
const current = exchanges(history.slice(lastUser + 1))
const decision = isWorkflow
? workflowDecision(current, user.content)
: receiptDecision(current, reusableDescription(history.slice(0, lastUser)))
const { stage } = decision
if (evidence.requests.length >= 512) throw new Error('evidence_capacity_exceeded')
evidence.completions++
// This fixture task needs one receipt tool. Other fixture catalog names
// anywhere in the payload reveal a catalog dump, including in messages.
const fixtureNames = new Set(
JSON.stringify(payload).match(/workitem_read_(?:receipt|\d{3})/g) ?? []
)
const connectorDefinitionCount = tools.filter(
tool =>
typeof tool.name === 'string' &&
/(?:^|__)workitem_read_(?:receipt|\d{3})$/.test(tool.name)
).length
evidence.requests.push({
definitionCount: tools.length,
explicitNonStrictCount: tools.filter(tool => tool.strict === false).length,
definitionBytes: Buffer.byteLength(JSON.stringify(tools)),
inputBytes: Buffer.byteLength(JSON.stringify(payload.input)),
connectorDefinitionCount,
leakedSchema: connectorDefinitionCount > 0 || fixtureNames.size > 1,
stage,
})
if (stage === 'denied') {
evidence.deniedResponses++
return stream({
type: 'response.output_text.delta',
delta: JSON.stringify(decision.result),
})
}
if (stage === 'final') {
evidence.finalResponses++
return stream({
type: 'response.output_text.delta',
delta: JSON.stringify(decision.result),
})
}
if (!names.has(stage)) throw new Error('unadvertised_function')
const args = decision.args
if (stage === BRIDGES[0]) evidence.searchCalls++
if (stage === BRIDGES[1]) evidence.describeCalls++
if (stage === BRIDGES[2]) evidence.businessCalls++
return stream({
type: 'response.output_item.done',
item: {
type: 'function_call',
id: randomUUID(),
call_id: randomUUID(),
name: stage,
arguments: JSON.stringify(args),
},
})
} catch {
evidence.rejected++
return Response.json({ error: { code: 'fixture_protocol_rejected' } }, { status: 422 })
}
}
return { fetchFn, evidence: () => structuredClone(evidence) }
}