Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { lstatSync, readdirSync, readFileSync } from 'node:fs'
import path from 'node:path'
import { pathToFileURL } from 'node:url'
export const namedRoots = [
'control-api', 'mcp-host', 'workflow-recipes', 'host-context-controller',
'control-ui', 'desktop-app', 'tests/e2e', 'scripts/e2e', 'scripts/tests',
]
const excludedDirectories = new Set([
'node_modules', 'dist', '.next', 'coverage', '.git', 'test-results',
'playwright-report', 'results', 'artifacts',
])
const testFile = /(?:\.test\.(?:ts|tsx|mjs|cjs)|\.spec\.(?:ts|tsx))$/i
function realPgLane(file) {
return file.startsWith('control-api/test/') && file.endsWith('.realPostgres.integration.test.ts')
? 'real-pg:control-api'
: undefined
}
export function isSubscriptionCandidate(file, provider) {
if (calibrationSuites.has(file)) return true
const name = path.posix.basename(file).toLowerCase()
const other = provider === 'codex' ? 'grok' : 'codex'
return name.includes(provider) ||
(!name.includes(other) && (name.includes('subscription') || name.includes('attachment')))
}
// These pre-existing shared suites have their own physical CI/PG consumers.
// New generic names do not inherit a lane: they must be registered explicitly.
const otherLaneSuites = new Map([
['control-api/test/subscriptionGrantIdentity.test.ts', 'general-ci:control-api'],
['control-api/test/config.subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],
['control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],
['control-api/test/services.subscriptionCatalogSyncCron.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/db.codexSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.codexSubscriptionOAuth.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.grokProviderAttemptRedemption.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/pluginWorkloadSdkGrokDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts', 'real-pg:control-api'],
['control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts', 'real-pg:control-api'],
['mcp-host/src/__tests__/server.attachments.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/agent/__tests__/attachmentRead.sqlite.integration.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/agent/__tests__/attachmentRead.integration.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/agent/__tests__/incomingAttachments.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/runtime/__tests__/fileAttachmentDelivery.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/core/orchestration/__tests__/toolUseLoopSingleTool.attachmentRead.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/core/tools/__tests__/nativeToolRegistry.attachmentRead.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/core/tools/__tests__/generatedArtifactAttachments.test.ts', 'general-ci:mcp-host'],
['mcp-host/src/core/tools/__tests__/attachmentRead.test.ts', 'general-ci:mcp-host'],
['desktop-app/ui/src/lib/__tests__/chatMessageAttachments.test.ts', 'general-ci:desktop-app'],
])
const calibrationSuites = new Map([
['scripts/tests/measure-control-api-authorize-memory.test.mjs', 'calibration:control-api-memory'],
])
const runtimeSuites = new Map([
['desktop-app/test/e2e-playwright/subscription-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts'],
['desktop-app/test/e2e-playwright/codex-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.codex-image.config.ts'],
['desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts', 'desktop-app/test/e2e-playwright/playwright.config.ts'],
['desktop-app/test/e2e-playwright/subscription-tool-screenshot.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-tool-screenshot.config.ts'],
['desktop-app/test/e2e-playwright/subscription-gfs-image.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-gfs-image.config.ts'],
['desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-admission-recovery.config.ts'],
['tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts', 'tests/e2e/playwright/playwright.subscription-admission.config.ts'],
['tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts', 'tests/e2e/playwright/playwright.codex-approved-tools.config.ts'],
['tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
['scripts/e2e/e2e-codex-subscription-runtime.sh', 'scripts/e2e/e2e-codex-subscription-runtime.sh'],
['scripts/e2e/e2e-codex-subscription-playwright.sh', 'scripts/e2e/e2e-codex-subscription-playwright.sh'],
['scripts/e2e/e2e-codex-subscription-network-boundary.sh', 'scripts/e2e/e2e-codex-subscription-network-boundary.sh'],
])
function regularFile(root, file) {
try { return lstatSync(path.join(root, file)).isFile() } catch { return false }
}
function walk(root, directory) {
const files = []
for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) {
const relative = `${directory}/${entry.name}`
if (entry.isDirectory() && !excludedDirectories.has(entry.name)) files.push(...walk(root, relative))
else if (entry.isFile()) files.push(relative)
}
return files
}
function isTest(file) {
return testFile.test(file) || /^e2e-.*\.sh$/i.test(path.posix.basename(file))
}
function configPatterns(source, key) {
const block = new RegExp(`${key}:\\s*\\[([\\s\\S]*?)\\]`).exec(source)?.[1] ?? ''
return [...block.matchAll(/['"]([^'"]+)['"]/g)].map(match => match[1])
}
function configSelectors(source, key) {
const values = []
const assignments = new RegExp(
`${key}:\\s*(\\[[^\\]]*\\]|/(?:\\\\.|[^/\\\\])+/[a-z]*|'[^']*'|"[^"]*")`,
'g',
)
for (const match of source.matchAll(assignments)) {
const value = match[1]
if (value.startsWith('[')) values.push(...configPatterns(`${key}: [${value.slice(1, -1)}]`, key))
else if (value.startsWith('/')) values.push(new RegExp(value.slice(1, value.lastIndexOf('/'))))
else values.push(value.slice(1, -1))
}
return values
}
function matchesPlaywrightSelector(relative, selector) {
if (selector instanceof RegExp) return selector.test(relative)
return path.matchesGlob(relative, selector) ||
(!selector.includes('/') && path.matchesGlob(path.posix.basename(relative), selector))
}
function projectBlocks(source) {
const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)
if (!projectsAssignment) return []
const arrayStart = source.indexOf('[', projectsAssignment.index)
const blocks = []
let depth = 0
let blockDepth = 0
let blockStart = -1
let quote
for (let index = arrayStart; index < source.length; index += 1) {
const character = source[index]
if (quote) {
if (character === '\\') index += 1
else if (character === quote) quote = undefined
continue
}
if (character === '\'' || character === '"' || character === '`') {
quote = character
continue
}
if (character === '[' || character === '{') {
depth += 1
if (character === '{' && depth === 2) {
blockDepth = depth
blockStart = index
}
continue
}
if (character === ']' || character === '}') {
depth -= 1
if (character === ']' && depth === 0) return blocks
}
if (character === '}') {
if (blockStart >= 0 && depth === 1) {
blocks.push(source.slice(blockStart, index + 1))
blockStart = -1
}
}
}
return blocks
}
function playwrightCollectsFile(source, configDirectory, file) {
if (source.includes('subscriptionRemainingJourneyConfig')) {
const delegatedRemainingConfig = /remainingJourneyConfig\(\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\)/.exec(source)
if (!delegatedRemainingConfig) return false
const helper = readFileSync(path.join(configDirectory, 'subscriptionRemainingJourneyConfig.ts'), 'utf8')
return path.posix.basename(file) === delegatedRemainingConfig[2] &&
/return\s+defineConfig\s*\(/.test(helper) &&
helper.includes('testMatch: `**/${specFile}`') &&
/=\s*requireSubscriptionImageRun\s*\(/.test(helper) &&
/(?:^|\n)\s*requireRemainingJourney\s*\(/.test(helper)
}
if (!source.includes('defineConfig')) return false
const relative = path.posix.relative(configDirectory, file)
const testDirectory = /(?:^|[{,\n]\s*)testDir:\s*['"](.+)['"]/.exec(source)?.[1]
if (testDirectory) {
const normalized = path.posix.normalize(testDirectory)
if (normalized !== '.' && !relative.startsWith(`${normalized}/`)) return false
}
const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)
const topLevelSource = projectsAssignment
? source.slice(0, projectsAssignment.index)
: source
const ignored = configSelectors(topLevelSource, 'testIgnore')
.some(selector => matchesPlaywrightSelector(relative, selector))
if (ignored) return false
const topLevelSelectors = configSelectors(topLevelSource, 'testMatch')
if (topLevelSelectors.length > 0) {
return topLevelSelectors.some(selector => matchesPlaywrightSelector(relative, selector))
}
const blocks = projectBlocks(source)
if (blocks.length === 0) return true
return blocks.some(block => {
const blockIgnored = configSelectors(block, 'testIgnore')
.some(selector => matchesPlaywrightSelector(relative, selector))
const selectors = configSelectors(block, 'testMatch')
if (selectors.length === 0) return !blockIgnored
return !blockIgnored &&
selectors.some(selector => matchesPlaywrightSelector(relative, selector))
})
}
export function verifyRuntimeConsumer(root, file, consumer) {
if (!regularFile(root, file) || !regularFile(root, consumer)) return undefined
if (/\.sh$/i.test(consumer)) {
const source = readFileSync(path.join(root, consumer), 'utf8')
const executable = (lstatSync(path.join(root, consumer)).mode & 0o111) !== 0
if (consumer !== file || !source.startsWith('#!') || !executable) return undefined
return `${consumer} (explicit self entrypoint; separate physical receipt required)`
}
const source = readFileSync(path.join(root, consumer), 'utf8')
const configDirectory = path.join(root, path.posix.dirname(consumer))
if (!playwrightCollectsFile(source, configDirectory, path.join(root, file))) return undefined
return `${consumer} (collection-verified registry entry)`
}
function verifyOtherConsumer(root, file, lane) {
const service = lane.split(':')[1]
const ciPath = '.github/workflows/ci-public.yml'
const configPath = `${service}/vitest.config.ts`
if (!regularFile(root, ciPath) || !regularFile(root, configPath)) return undefined
const ci = readFileSync(path.join(root, ciPath), 'utf8')
const config = readFileSync(path.join(root, configPath), 'utf8')
const relative = file.slice(service.length + 1)
const included = configPatterns(config, 'include').some(pattern => path.matchesGlob(relative, pattern))
const excluded = configPatterns(config, 'exclude').some(pattern => path.matchesGlob(relative, pattern))
if (!included || excluded) return undefined
if (lane.startsWith('real-pg:')) {
const pgJob = ci.split(' control-api-migration:')[1]?.split(/\n [a-z][a-z0-9-]*:/)[0]
if (
!pgJob ||
!/CONTROL_API_REAL_PG_REQUIRED:\s*'1'/.test(pgJob) ||
!/npm test.*(?:--run )?realPostgres/.test(pgJob)
) return undefined
return `${ciPath}#control-api-migration (realPostgres filter); ${configPath}`
}
if (!new RegExp(`^\\s*- ${service}\\s*$`, 'm').test(ci) || !ci.includes('npm test')) return undefined
return `${ciPath}#test (${service}); ${configPath}`
}
const nodeTestRunner = 'scripts/tests/run-node-test-files.mjs'
// A workflow step consumes `file` only when its own run command passes it to
// the explicit node:test runner from the repository root. A comment, another
// step's command, a relocated working-directory or a step allowed to fail
// does not count.
function ciStepRunsNodeTest(ci, file) {
return ci.split(/\n\s*- (?=name:|uses:|run:)/).slice(1).some(step => {
if (/^\s*(?:working-directory|continue-on-error):/m.test(step)) return false
const run = step.split(/^\s*run:/m)[1]
if (run === undefined) return false
const tokens = run.split('\n')
.filter(line => !line.trim().startsWith('#'))
.join(' ')
.replace(/\\(?=\s|$)/g, ' ')
.split(/\s+/)
.filter(token => token && token !== '|')
const runner = tokens.findIndex((token, index) => token === nodeTestRunner && tokens[index - 1] === 'node')
return runner !== -1 && tokens.slice(runner + 1).includes(file)
})
}
function verifyCalibrationConsumer(root, file, lane) {
if (lane !== 'calibration:control-api-memory') return undefined
const driver = 'scripts/tests/measure-control-api-authorize-memory.mjs'
const ciPath = '.github/workflows/ci-public.yml'
if (!regularFile(root, file) || !regularFile(root, driver) || !regularFile(root, ciPath)) return undefined
const makefile = regularFile(root, 'Makefile')
? readFileSync(path.join(root, 'Makefile'), 'utf8')
: ''
if (!makefile.includes('minikube-control-api-authorize-memory:')) return undefined
if (!ciStepRunsNodeTest(readFileSync(path.join(root, ciPath), 'utf8'), file)) return undefined
return `${ciPath} (${nodeTestRunner} ${file}); Makefile#minikube-control-api-authorize-memory (physical calibration only); ${driver}`
}
export function auditSubscriptionDiscovery(provider, root, registered) {
if (!['grok', 'codex'].includes(provider)) throw new Error('provider must be grok or codex')
const suiteRoots = provider === 'grok'
? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']
: ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']
const violations = [], candidates = [], roots = [...suiteRoots, ...namedRoots]
const filesByRoot = new Map()
const registeredSet = new Set(registered)
if (registeredSet.size !== registered.length) violations.push('duplicate registered physical suite')
for (const file of registered) {
if (path.posix.isAbsolute(file) || path.posix.normalize(file) !== file || file.startsWith('../') || !regularFile(root, file)) {
violations.push(`missing or invalid registered suite ${file}`)
}
}
for (const directory of roots) {
let valid = false
try { valid = lstatSync(path.join(root, directory)).isDirectory() } catch { /* Missing is a violation. */ }
if (!valid) { violations.push(`missing discovery root ${directory}`); continue }
const files = walk(root, directory).filter(isTest)
if (files.length === 0) violations.push(`empty discovery root ${directory}`)
filesByRoot.set(directory, files)
}
for (const [directory, files] of filesByRoot) {
for (const file of files) {
if (!suiteRoots.includes(directory) && !isSubscriptionCandidate(file, provider)) continue
let lane, consumer
const pgLane = realPgLane(file)
const pgRegistered = Boolean(pgLane) &&
registeredSet.has(file) && otherLaneSuites.get(file) === pgLane
const calibrationLane = calibrationSuites.get(file)
const calibrationRegistered = registeredSet.has(file) && calibrationLane !== undefined
if (runtimeSuites.has(file)) {
lane = 'runtime-opt-in'
if (registeredSet.has(file)) {
violations.push(`runtime suite cannot be registered as a T0 unit ${file}`)
continue
}
consumer = verifyRuntimeConsumer(root, file, runtimeSuites.get(file))
if (!consumer) violations.push(`runtime consumer does not collect ${file}`)
} else if (pgRegistered) {
lane = pgLane
consumer = verifyOtherConsumer(root, file, pgLane)
if (!consumer) violations.push(`unverified ${pgLane} consumer for ${file}`)
else violations.push(`real-PG suite cannot be registered as a T0 unit ${file}`)
} else if (calibrationRegistered) {
lane = calibrationLane
consumer = verifyCalibrationConsumer(root, file, calibrationLane)
if (!consumer) violations.push(`unverified ${calibrationLane} consumer for ${file}`)
else violations.push(`calibration suite cannot be registered as a T0 unit ${file}`)
} else if (registeredSet.has(file)) {
lane = 'T0'
consumer = 'physical T0 group; separate producer/reporter required'
} else if (pgLane && otherLaneSuites.get(file) === pgLane) {
lane = pgLane
consumer = verifyOtherConsumer(root, file, lane)
if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)
} else if (calibrationLane !== undefined) {
lane = calibrationLane
consumer = verifyCalibrationConsumer(root, file, lane)
if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)
} else if (otherLaneSuites.has(file)) {
lane = otherLaneSuites.get(file)
consumer = verifyOtherConsumer(root, file, lane)
if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)
} else {
violations.push(`unlisted ${provider} subscription suite ${file}`)
lane = 'UNLISTED'
}
candidates.push({ file, lane, consumer, evidence: 'NOT_RUN_HERE', requiredPhysicalEvidence: true })
}
}
if (candidates.length === 0) violations.push(`empty ${provider} subscription candidate scan`)
return { provider, roots, candidates: candidates.sort((a, b) => a.file.localeCompare(b.file)), violations }
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const [provider, root, registryFile, outputMode] = process.argv.slice(2)
if (!provider || !root || !registryFile || (outputMode && outputMode !== '--json')) {
console.error('usage: subscription-t0-discovery.mjs <grok|codex> <root> <registered-file> [--json]')
process.exit(2)
}
const registered = readFileSync(registryFile, 'utf8').split(/\r?\n/).filter(Boolean)
const result = auditSubscriptionDiscovery(provider, root, registered)
if (outputMode === '--json') console.log(JSON.stringify(result))
else {
for (const entry of result.candidates.filter(entry => entry.lane !== 'T0')) {
console.log(`INVENTORY ${entry.lane} NOT_RUN_HERE: ${entry.file}; physical evidence required via ${entry.consumer ?? 'explicit registration'}`)
}
for (const violation of result.violations) console.error(`FAIL: ${violation}`)
console.log(`Discovery inventory: ${provider}, ${result.candidates.length} candidates, ${result.violations.length} violations; execution evidence is separate`)
}
process.exit(result.violations.length ? 1 : 0)
}