Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(llm): verify complete subscription test discovery

Alfredo Lopez76bfa59b9224
diff --git a/scripts/tests/lib/subscription-t0-discovery.mjs b/scripts/tests/lib/subscription-t0-discovery.mjs
new file mode 100644
index 000000000..d6f8a14ca
--- /dev/null
+++ b/scripts/tests/lib/subscription-t0-discovery.mjs
@@ -0,0 +1,313 @@
+import { lstatSync, readdirSync, readFileSync } from 'node:fs'
+import path from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+export const namedRoots = [
+ 'control-api', 'mcp-host', 'workflow-recipes', 'host-context-controller',
+ 'control-ui', 'desktop-app', 'tests/e2e', 'scripts/e2e',
+]
+const excludedDirectories = new Set([
+ 'node_modules', 'dist', '.next', 'coverage', '.git', 'test-results',
+ 'playwright-report', 'results', 'artifacts',
+])
+const testFile = /(?:\.test\.(?:ts|tsx|mjs|cjs)|\.spec\.(?:ts|tsx))$/i
+
+function realPgLane(file) {
+ return file.startsWith('control-api/test/') && file.endsWith('.realPostgres.integration.test.ts')
+ ? 'real-pg:control-api'
+ : undefined
+}
+
+export function isSubscriptionCandidate(file, provider) {
+ const name = path.posix.basename(file).toLowerCase()
+ const other = provider === 'codex' ? 'grok' : 'codex'
+ return name.includes(provider) ||
+ (!name.includes(other) && (name.includes('subscription') || name.includes('attachment')))
+}
+
+// These pre-existing shared suites have their own physical CI/PG consumers.
+// New generic names do not inherit a lane: they must be registered explicitly.
+const otherLaneSuites = new Map([
+ ['control-api/test/subscriptionGrantIdentity.test.ts', 'general-ci:control-api'],
+ ['control-api/test/config.subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],
+ ['control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],
+ ['control-api/test/services.subscriptionCatalogSyncCron.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/db.codexSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.codexSubscriptionOAuth.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.grokProviderAttemptRedemption.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/pluginWorkloadSdkGrokDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts', 'real-pg:control-api'],
+ ['mcp-host/src/__tests__/server.attachments.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/agent/__tests__/attachmentRead.sqlite.integration.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/agent/__tests__/attachmentRead.integration.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/agent/__tests__/incomingAttachments.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/runtime/__tests__/fileAttachmentDelivery.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/core/orchestration/__tests__/toolUseLoopSingleTool.attachmentRead.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/core/tools/__tests__/nativeToolRegistry.attachmentRead.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/core/tools/__tests__/generatedArtifactAttachments.test.ts', 'general-ci:mcp-host'],
+ ['mcp-host/src/core/tools/__tests__/attachmentRead.test.ts', 'general-ci:mcp-host'],
+ ['desktop-app/ui/src/lib/__tests__/chatMessageAttachments.test.ts', 'general-ci:desktop-app'],
+])
+const runtimeSuites = new Map([
+ ['desktop-app/test/e2e-playwright/subscription-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts'],
+ ['desktop-app/test/e2e-playwright/codex-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.codex-image.config.ts'],
+ ['desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts', 'desktop-app/test/e2e-playwright/playwright.config.ts'],
+ ['tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts', 'tests/e2e/playwright/playwright.subscription-admission.config.ts'],
+ ['tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts', 'tests/e2e/playwright/playwright.codex-approved-tools.config.ts'],
+ ['tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],
+ ['scripts/e2e/e2e-codex-subscription-runtime.sh', 'scripts/e2e/e2e-codex-subscription-runtime.sh'],
+ ['scripts/e2e/e2e-codex-subscription-playwright.sh', 'scripts/e2e/e2e-codex-subscription-playwright.sh'],
+ ['scripts/e2e/e2e-codex-subscription-network-boundary.sh', 'scripts/e2e/e2e-codex-subscription-network-boundary.sh'],
+])
+
+function regularFile(root, file) {
+ try { return lstatSync(path.join(root, file)).isFile() } catch { return false }
+}
+function walk(root, directory) {
+ const files = []
+ for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) {
+ const relative = `${directory}/${entry.name}`
+ if (entry.isDirectory() && !excludedDirectories.has(entry.name)) files.push(...walk(root, relative))
+ else if (entry.isFile()) files.push(relative)
+ }
+ return files
+}
+function isTest(file) {
+ return testFile.test(file) || /^e2e-.*\.sh$/i.test(path.posix.basename(file))
+}
+function configPatterns(source, key) {
+ const block = new RegExp(`${key}:\\s*\\[([\\s\\S]*?)\\]`).exec(source)?.[1] ?? ''
+ return [...block.matchAll(/['"]([^'"]+)['"]/g)].map(match => match[1])
+}
+
+function configSelectors(source, key) {
+ const values = []
+ const assignments = new RegExp(
+ `${key}:\\s*(\\[[^\\]]*\\]|/(?:\\\\.|[^/\\\\])+/[a-z]*|'[^']*'|"[^"]*")`,
+ 'g',
+ )
+ for (const match of source.matchAll(assignments)) {
+ const value = match[1]
+ if (value.startsWith('[')) values.push(...configPatterns(`${key}: [${value.slice(1, -1)}]`, key))
+ else if (value.startsWith('/')) values.push(new RegExp(value.slice(1, value.lastIndexOf('/'))))
+ else values.push(value.slice(1, -1))
+ }
+ return values
+}
+
+function matchesPlaywrightSelector(relative, selector) {
+ if (selector instanceof RegExp) return selector.test(relative)
+ return path.matchesGlob(relative, selector) ||
+ (!selector.includes('/') && path.matchesGlob(path.posix.basename(relative), selector))
+}
+
+function projectBlocks(source) {
+ const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)
+ if (!projectsAssignment) return []
+ const arrayStart = source.indexOf('[', projectsAssignment.index)
+ const blocks = []
+ let depth = 0
+ let blockDepth = 0
+ let blockStart = -1
+ let quote
+ for (let index = arrayStart; index < source.length; index += 1) {
+ const character = source[index]
+ if (quote) {
+ if (character === '\\') index += 1
+ else if (character === quote) quote = undefined
+ continue
+ }
+ if (character === '\'' || character === '"' || character === '`') {
+ quote = character
+ continue
+ }
+ if (character === '[' || character === '{') {
+ depth += 1
+ if (character === '{' && depth === 2) {
+ blockDepth = depth
+ blockStart = index
+ }
+ continue
+ }
+ if (character === ']' || character === '}') {
+ depth -= 1
+ if (character === ']' && depth === 0) return blocks
+ }
+ if (character === '}') {
+ if (blockStart >= 0 && depth === 1) {
+ blocks.push(source.slice(blockStart, index + 1))
+ blockStart = -1
+ }
+ }
+ }
+ return blocks
+}
+
+function playwrightCollectsFile(source, configDirectory, file) {
+ if (!source.includes('defineConfig')) return false
+ const relative = path.posix.relative(configDirectory, file)
+ const testDirectory = /(?:^|[{,\n]\s*)testDir:\s*['"](.+)['"]/.exec(source)?.[1]
+ if (testDirectory) {
+ const normalized = path.posix.normalize(testDirectory)
+ if (normalized !== '.' && !relative.startsWith(`${normalized}/`)) return false
+ }
+
+ const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)
+ const topLevelSource = projectsAssignment
+ ? source.slice(0, projectsAssignment.index)
+ : source
+ const ignored = configSelectors(topLevelSource, 'testIgnore')
+ .some(selector => matchesPlaywrightSelector(relative, selector))
+ if (ignored) return false
+
+ const topLevelSelectors = configSelectors(topLevelSource, 'testMatch')
+ if (topLevelSelectors.length > 0) {
+ return topLevelSelectors.some(selector => matchesPlaywrightSelector(relative, selector))
+ }
+
+ const blocks = projectBlocks(source)
+ if (blocks.length === 0) return true
+ return blocks.some(block => {
+ const blockIgnored = configSelectors(block, 'testIgnore')
+ .some(selector => matchesPlaywrightSelector(relative, selector))
+ const selectors = configSelectors(block, 'testMatch')
+ if (selectors.length === 0) return !blockIgnored
+ return !blockIgnored &&
+ selectors.some(selector => matchesPlaywrightSelector(relative, selector))
+ })
+}
+
+export function verifyRuntimeConsumer(root, file, consumer) {
+ if (!regularFile(root, file) || !regularFile(root, consumer)) return undefined
+
+ if (/\.sh$/i.test(consumer)) {
+ const source = readFileSync(path.join(root, consumer), 'utf8')
+ const executable = (lstatSync(path.join(root, consumer)).mode & 0o111) !== 0
+ if (consumer !== file || !source.startsWith('#!') || !executable) return undefined
+ return `${consumer} (explicit self entrypoint; separate physical receipt required)`
+ }
+
+ const source = readFileSync(path.join(root, consumer), 'utf8')
+ if (!playwrightCollectsFile(source, path.posix.dirname(consumer), file)) return undefined
+ return `${consumer} (collection-verified registry entry)`
+}
+
+function verifyOtherConsumer(root, file, lane) {
+ const service = lane.split(':')[1]
+ const ciPath = '.github/workflows/ci-public.yml'
+ const configPath = `${service}/vitest.config.ts`
+ if (!regularFile(root, ciPath) || !regularFile(root, configPath)) return undefined
+ const ci = readFileSync(path.join(root, ciPath), 'utf8')
+ const config = readFileSync(path.join(root, configPath), 'utf8')
+ const relative = file.slice(service.length + 1)
+ const included = configPatterns(config, 'include').some(pattern => path.matchesGlob(relative, pattern))
+ const excluded = configPatterns(config, 'exclude').some(pattern => path.matchesGlob(relative, pattern))
+ if (!included || excluded) return undefined
+ if (lane.startsWith('real-pg:')) {
+ const pgJob = ci.split(' control-api-migration:')[1]?.split(/\n [a-z][a-z0-9-]*:/)[0]
+ if (
+ !pgJob ||
+ !/CONTROL_API_REAL_PG_REQUIRED:\s*'1'/.test(pgJob) ||
+ !/npm test.*(?:--run )?realPostgres/.test(pgJob)
+ ) return undefined
+ return `${ciPath}#control-api-migration (realPostgres filter); ${configPath}`
+ }
+ if (!new RegExp(`^\\s*- ${service}\\s*$`, 'm').test(ci) || !ci.includes('npm test')) return undefined
+ return `${ciPath}#test (${service}); ${configPath}`
+}
+
+export function auditSubscriptionDiscovery(provider, root, registered) {
+ if (!['grok', 'codex'].includes(provider)) throw new Error('provider must be grok or codex')
+ const suiteRoots = provider === 'grok'
+ ? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']
+ : ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']
+ const violations = [], candidates = [], roots = [...suiteRoots, ...namedRoots]
+ const filesByRoot = new Map()
+ const registeredSet = new Set(registered)
+ if (registeredSet.size !== registered.length) violations.push('duplicate registered physical suite')
+ for (const file of registered) {
+ if (path.posix.isAbsolute(file) || path.posix.normalize(file) !== file || file.startsWith('../') || !regularFile(root, file)) {
+ violations.push(`missing or invalid registered suite ${file}`)
+ }
+ }
+ for (const directory of roots) {
+ let valid = false
+ try { valid = lstatSync(path.join(root, directory)).isDirectory() } catch { /* Missing is a violation. */ }
+ if (!valid) { violations.push(`missing discovery root ${directory}`); continue }
+ const files = walk(root, directory).filter(isTest)
+ if (files.length === 0) violations.push(`empty discovery root ${directory}`)
+ filesByRoot.set(directory, files)
+ }
+ for (const [directory, files] of filesByRoot) {
+ for (const file of files) {
+ if (!suiteRoots.includes(directory) && !isSubscriptionCandidate(file, provider)) continue
+ let lane, consumer
+ const pgLane = realPgLane(file)
+ const pgRegistered = Boolean(pgLane) &&
+ registeredSet.has(file) && otherLaneSuites.get(file) === pgLane
+ if (runtimeSuites.has(file)) {
+ lane = 'runtime-opt-in'
+ if (registeredSet.has(file)) {
+ violations.push(`runtime suite cannot be registered as a T0 unit ${file}`)
+ continue
+ }
+ consumer = verifyRuntimeConsumer(root, file, runtimeSuites.get(file))
+ if (!consumer) violations.push(`runtime consumer does not collect ${file}`)
+ } else if (pgRegistered) {
+ lane = pgLane
+ consumer = verifyOtherConsumer(root, file, pgLane)
+ if (!consumer) violations.push(`unverified ${pgLane} consumer for ${file}`)
+ else violations.push(`real-PG suite cannot be registered as a T0 unit ${file}`)
+ } else if (registeredSet.has(file)) {
+ lane = 'T0'
+ consumer = 'physical T0 group; separate producer/reporter required'
+ } else if (pgLane && otherLaneSuites.get(file) === pgLane) {
+ lane = pgLane
+ consumer = verifyOtherConsumer(root, file, lane)
+ if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)
+ } else if (otherLaneSuites.has(file)) {
+ lane = otherLaneSuites.get(file)
+ consumer = verifyOtherConsumer(root, file, lane)
+ if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)
+ } else {
+ violations.push(`unlisted ${provider} subscription suite ${file}`)
+ lane = 'UNLISTED'
+ }
+ candidates.push({ file, lane, consumer, evidence: 'NOT_RUN_HERE', requiredPhysicalEvidence: true })
+ }
+ }
+ if (candidates.length === 0) violations.push(`empty ${provider} subscription candidate scan`)
+ return { provider, roots, candidates: candidates.sort((a, b) => a.file.localeCompare(b.file)), violations }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [provider, root, registryFile, outputMode] = process.argv.slice(2)
+ if (!provider || !root || !registryFile || (outputMode && outputMode !== '--json')) {
+ console.error('usage: subscription-t0-discovery.mjs <grok|codex> <root> <registered-file> [--json]')
+ process.exit(2)
+ }
+ const registered = readFileSync(registryFile, 'utf8').split(/\r?\n/).filter(Boolean)
+ const result = auditSubscriptionDiscovery(provider, root, registered)
+ if (outputMode === '--json') console.log(JSON.stringify(result))
+ else {
+ for (const entry of result.candidates.filter(entry => entry.lane !== 'T0')) {
+ console.log(`INVENTORY ${entry.lane} NOT_RUN_HERE: ${entry.file}; physical evidence required via ${entry.consumer ?? 'explicit registration'}`)
+ }
+ for (const violation of result.violations) console.error(`FAIL: ${violation}`)
+ console.log(`Discovery inventory: ${provider}, ${result.candidates.length} candidates, ${result.violations.length} violations; execution evidence is separate`)
+ }
+ process.exit(result.violations.length ? 1 : 0)
+}
diff --git a/scripts/tests/subscription-t0-discovery.test.mjs b/scripts/tests/subscription-t0-discovery.test.mjs
new file mode 100644
index 000000000..313199621
--- /dev/null
+++ b/scripts/tests/subscription-t0-discovery.test.mjs
@@ -0,0 +1,311 @@
+import assert from 'node:assert/strict'
+import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+import {
+ auditSubscriptionDiscovery,
+ isSubscriptionCandidate,
+ namedRoots,
+ verifyRuntimeConsumer,
+} from './lib/subscription-t0-discovery.mjs'
+
+function suiteRoots(provider) {
+ return provider === 'grok'
+ ? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']
+ : ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']
+}
+
+function createFile(root, relative, content = '') {
+ const target = path.join(root, relative)
+ mkdirSync(path.dirname(target), { recursive: true })
+ writeFileSync(target, content)
+ return relative
+}
+
+function makeFixture(t, provider = 'codex') {
+ const root = mkdtempSync(path.join(tmpdir(), `subscription-discovery-${provider}-`))
+ t.after(() => rmSync(root, { recursive: true, force: true }))
+ const registered = []
+ for (const directory of [...suiteRoots(provider), ...namedRoots]) {
+ mkdirSync(path.join(root, directory), { recursive: true })
+ }
+ for (const [index, directory] of suiteRoots(provider).entries()) {
+ const file = `${directory}/${provider}-suite-${index}.test.ts`
+ createFile(root, file)
+ registered.push(file)
+ }
+ namedRoots.forEach((directory, index) => {
+ createFile(root, `${directory}/baseline-${index}.test.ts`)
+ })
+ return { root, registered }
+}
+
+test('admits the three formerly omitted roots through their physical file entries', t => {
+ const fixture = makeFixture(t)
+ const desktop = createFile(
+ fixture.root,
+ 'desktop-app/ui/src/constants/__tests__/attachments.test.ts',
+ )
+ const host = createFile(
+ fixture.root,
+ 'mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts',
+ )
+ const e2e = createFile(
+ fixture.root,
+ 'tests/e2e/integration/codex-subscription-contract-freeze.test.ts',
+ )
+ const scripts = createFile(
+ fixture.root,
+ 'scripts/e2e/prepare-codex-approved-tools.test.mjs',
+ )
+ const registered = [...fixture.registered, desktop, host, e2e, scripts]
+ const result = auditSubscriptionDiscovery('codex', fixture.root, registered)
+
+ assert.deepEqual(result.violations, [])
+ const lanes = new Map(result.candidates.map(entry => [entry.file, entry.lane]))
+ for (const file of [desktop, host, e2e, scripts]) assert.equal(lanes.get(file), 'T0')
+ for (const root of ['desktop-app', 'tests/e2e', 'scripts/e2e']) {
+ assert.ok(result.roots.includes(root))
+ }
+})
+
+test('generic attachment candidates belong to both provider lanes; provider names stay isolated', () => {
+ const attachment = 'any/deep/attachments.test.ts'
+ const codex = 'mcp-host/src/llm/__tests__/codexSubscription.test.ts'
+ const grok = 'mcp-host/src/llm/__tests__/grokSubscription.test.ts'
+ assert.equal(isSubscriptionCandidate(attachment, 'codex'), true)
+ assert.equal(isSubscriptionCandidate(attachment, 'grok'), true)
+ assert.equal(isSubscriptionCandidate(codex, 'codex'), true)
+ assert.equal(isSubscriptionCandidate(codex, 'grok'), false)
+ assert.equal(isSubscriptionCandidate(grok, 'grok'), true)
+ assert.equal(isSubscriptionCandidate(grok, 'codex'), false)
+})
+
+test('missing and empty discovery roots fail with their exact causal violation', t => {
+ const missing = makeFixture(t)
+ rmSync(path.join(missing.root, 'tests/e2e'), { recursive: true, force: true })
+ const missingResult = auditSubscriptionDiscovery('codex', missing.root, missing.registered)
+ assert.ok(missingResult.violations.includes('missing discovery root tests/e2e'))
+
+ const empty = makeFixture(t, 'grok')
+ rmSync(path.join(empty.root, 'mcp-host/baseline-1.test.ts'))
+ const emptyResult = auditSubscriptionDiscovery('grok', empty.root, empty.registered)
+ assert.ok(emptyResult.violations.includes('empty discovery root mcp-host'))
+})
+
+test('duplicate and missing registered physical suites fail closed', t => {
+ const fixture = makeFixture(t)
+ const duplicateResult = auditSubscriptionDiscovery(
+ 'codex',
+ fixture.root,
+ [...fixture.registered, fixture.registered[0]],
+ )
+ assert.ok(duplicateResult.violations.includes('duplicate registered physical suite'))
+
+ const missingRelative = 'control-api/test/missing-registered-suite.test.ts'
+ createFile(path.dirname(path.join(fixture.root, missingRelative)), path.basename(missingRelative))
+ rmSync(path.join(fixture.root, missingRelative))
+ const missingResult = auditSubscriptionDiscovery(
+ 'codex',
+ fixture.root,
+ [...fixture.registered, missingRelative],
+ )
+ assert.ok(
+ missingResult.violations.includes(`missing or invalid registered suite ${missingRelative}`),
+ )
+})
+
+test('an unlisted candidate is UNLISTED even when another candidate is registered', t => {
+ const fixture = makeFixture(t)
+ const unlisted = createFile(
+ fixture.root,
+ 'mcp-host/src/llm/__tests__/newCodexSurface.test.ts',
+ )
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.ok(result.violations.includes(`unlisted codex subscription suite ${unlisted}`))
+ const entry = result.candidates.find(candidate => candidate.file === unlisted)
+ assert.equal(entry?.lane, 'UNLISTED')
+})
+
+test('registered real-PG suites are never confused with executed T0 suites', t => {
+ const fixture = makeFixture(t)
+ const suite = 'control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts'
+ createFile(fixture.root, suite)
+ createFile(
+ fixture.root,
+ 'control-api/vitest.config.ts',
+ "export default { include: ['test/**/*.test.ts'] }\n",
+ )
+ createFile(
+ fixture.root,
+ '.github/workflows/ci-public.yml',
+ ` control-api-migration:
+ steps:
+ - run: |
+ CONTROL_API_REAL_PG_REQUIRED: '1'
+ npm test -- --run realPostgres
+`,
+ )
+
+ const registeredResult = auditSubscriptionDiscovery(
+ 'codex',
+ fixture.root,
+ [...fixture.registered, suite],
+ )
+ const registeredEntry = registeredResult.candidates.find(entry => entry.file === suite)
+ assert.equal(registeredEntry?.lane, 'real-pg:control-api')
+ assert.ok(
+ registeredResult.violations.includes(`real-PG suite cannot be registered as a T0 unit ${suite}`),
+ )
+
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.deepEqual(result.violations, [])
+ const entry = result.candidates.find(candidate => candidate.file === suite)
+ assert.equal(entry?.lane, 'real-pg:control-api')
+ assert.match(String(entry?.consumer), /control-api-migration/)
+})
+
+test('a realPostgres suffix alone is not a silent other-lane registry', t => {
+ const fixture = makeFixture(t)
+ const unknown = createFile(
+ fixture.root,
+ 'control-api/test/unknownSubscription.realPostgres.integration.test.ts',
+ )
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.ok(result.violations.includes(`unlisted codex subscription suite ${unknown}`))
+})
+
+test('general CI lanes verify both matrix ownership and config collection', t => {
+ const fixture = makeFixture(t)
+ const suite = 'control-api/test/subscriptionGrantIdentity.test.ts'
+ const config = 'control-api/vitest.config.ts'
+ const workflow = '.github/workflows/ci-public.yml'
+ createFile(fixture.root, suite)
+ createFile(fixture.root, workflow, `jobs:
+ test:
+ strategy:
+ matrix:
+ service:
+ - control-api
+ steps:
+ - run: npm test
+`)
+ createFile(fixture.root, config, "export default { include: ['test/**/*.test.ts'] }\n")
+
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.deepEqual(result.violations, [])
+ const entry = result.candidates.find(candidate => candidate.file === suite)
+ assert.equal(entry?.lane, 'general-ci:control-api')
+ assert.match(String(entry?.consumer), /#test \(control-api\)/)
+
+ writeFileSync(path.join(fixture.root, config), "export default { include: ['test/other.test.ts'] }\n")
+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.ok(negative.violations.includes(`unverified general-ci:control-api consumer for ${suite}`))
+})
+
+test('Playwright runtime lanes require the registered config to collect the exact spec', t => {
+ const fixture = makeFixture(t)
+ const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'
+ const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'
+ createFile(fixture.root, spec)
+ createFile(
+ fixture.root,
+ config,
+ `import { defineConfig } from '@playwright/test'
+export default defineConfig({
+ testDir: '.',
+ testMatch: 'control-ui/codex-subscription-admission.spec.ts',
+})
+`,
+ )
+
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.deepEqual(result.violations, [])
+ const entry = result.candidates.find(candidate => candidate.file === spec)
+ assert.equal(entry?.lane, 'runtime-opt-in')
+ assert.match(String(entry?.consumer), /collection-verified registry entry/)
+
+ writeFileSync(
+ path.join(fixture.root, config),
+ `import { defineConfig } from '@playwright/test'
+export default defineConfig({
+ testDir: '.',
+ testMatch: 'control-ui/other.spec.ts',
+})
+`,
+ )
+ assert.equal(verifyRuntimeConsumer(fixture.root, spec, config), undefined)
+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.ok(negative.violations.includes(`runtime consumer does not collect ${spec}`))
+})
+
+test('shell runtime entries are explicit self entrypoints and reject non-entrypoint files', t => {
+ const fixture = makeFixture(t)
+ const script = 'scripts/e2e/e2e-codex-subscription-runtime.sh'
+ createFile(fixture.root, script, '#!/usr/bin/env bash\nset -euo pipefail\n')
+ chmodSync(path.join(fixture.root, script), 0o755)
+
+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.deepEqual(result.violations, [])
+ const entry = result.candidates.find(candidate => candidate.file === script)
+ assert.equal(entry?.lane, 'runtime-opt-in')
+ assert.match(String(entry?.consumer), /explicit self entrypoint/)
+
+ chmodSync(path.join(fixture.root, script), 0o644)
+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
+ assert.ok(negative.violations.includes(`runtime consumer does not collect ${script}`))
+})
+
+test('a Playwright default project can collect a spec outside another project selector', t => {
+ const fixture = makeFixture(t)
+ const spec = 'desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts'
+ const config = 'desktop-app/test/e2e-playwright/playwright.config.ts'
+ createFile(fixture.root, spec)
+ createFile(
+ fixture.root,
+ config,
+ `import { defineConfig } from '@playwright/test'
+export default defineConfig({
+ testDir: '.',
+ projects: [
+ {
+ name: 'default',
+ testIgnore: ['**/other-lane.spec.ts'],
+ },
+ {
+ name: 'special',
+ testMatch: /other-lane\\.spec\\.ts/,
+ },
+ ],
+})
+`,
+ )
+
+ assert.ok(verifyRuntimeConsumer(fixture.root, spec, config))
+})
+
+test('runtime suites cannot enter the T0 registry', t => {
+ const fixture = makeFixture(t)
+ const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'
+ const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'
+ createFile(fixture.root, spec)
+ createFile(
+ fixture.root,
+ config,
+ `import { defineConfig } from '@playwright/test'
+export default defineConfig({
+ testDir: '.',
+ testMatch: 'control-ui/codex-subscription-admission.spec.ts',
+})
+`,
+ )
+ const result = auditSubscriptionDiscovery(
+ 'codex',
+ fixture.root,
+ [...fixture.registered, spec],
+ )
+ assert.ok(
+ result.violations.includes(`runtime suite cannot be registered as a T0 unit ${spec}`),
+ )
+})
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 1a6b2d4a7..289c00547 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -14,6 +14,20 @@ FAIL=0
GROUPS_RUN=0
REGISTERED=()
+# Read-only inventory mode for hermetic discovery tests and operator inspection.
+# It does not enter package, cluster, Electron or browser execution.
+if [[ "${1:-}" == "--discovery-only" ]]; then
+ if [[ "$#" -ne 2 ]]; then
+ echo "usage: $0 --discovery-only <registered-file>" >&2
+ exit 2
+ fi
+ node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" codex "${ROOT}" "$2"
+ exit "$?"
+elif [[ "$#" -ne 0 ]]; then
+ echo "unsupported T0 argument: $1" >&2
+ exit 2
+fi
+
pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }
@@ -180,7 +194,6 @@ require_ci_matrix_entry() {
# proves they exist and that the CI real-PG lane asserts each one ran.
require_real_pg_suite() {
local rel="$1"
- REGISTERED+=("${rel}")
require_file "${rel}" || return 1
local suite
# The real-PG lane lists each suite by its file name, `.test.ts` included.
@@ -221,7 +234,12 @@ fi
run_group "shared-contract" "packages/llm-provider-attempt-contract" "index.test.cjs"
run_group "codex-catalog-projection" "packages/codex-catalog-projection" "index.test.cjs"
+# This is a filesystem-only freeze contract, not a browser/runtime journey.
+run_group "codex-upstream-contract-freeze" "tests/e2e" "integration/codex-subscription-contract-freeze.test.ts"
+
run_node_group "approved-tools-fixtures-and-runner" \
+ "scripts/tests/subscription-t0-discovery.test.mjs" \
+ "scripts/e2e/fixtures/subscription-image-provider.test.mjs" \
"scripts/tests/run-node-test-files.test.mjs" \
"tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs" \
"scripts/e2e/prepare-codex-approved-tools.test.mjs" \
@@ -310,6 +328,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \
run_group "mcp-host" "mcp-host" \
"src/__tests__/bodylimits.test.ts" \
+ "src/llm/__tests__/attachmentBudgetRefusal.test.ts" \
"src/llm/__tests__/imageSource.test.ts" \
"src/capabilities/toolCatalogTools.test.ts" \
"src/core/orchestration/__tests__/approvedToolsLifecycle.integration.test.ts" \
@@ -399,62 +418,28 @@ else
"src/__tests__/devIsolation.test.ts" \
"ui/src/components/agents/__tests__/ComposerPanel.test.tsx" \
"ui/src/components/agents/__tests__/ModelSelector.test.tsx" \
+ "test/subscriptionImageCollection.test.ts" \
+ "test/subscriptionImageChallenge.test.ts" \
+ "test/subscriptionImageRunContract.test.ts" \
+ "test/subscriptionAdmissionGuard.test.ts" \
+ "test/codexImageChallenge.test.ts" \
"ui/src/constants/__tests__/attachments.test.ts" \
"ui/src/hooks/__tests__/useHostModels.test.tsx" \
"ui/src/hooks/domain/__tests__/useAgentChatController.pendingModel.test.tsx"
fi
fi
-# A Codex suite that exists but is not listed above is lost coverage. Every
-# proxy/contract test file and every *codex* test file in the Codex-touching
-# packages must be registered in a group or as a real-PG presence check.
-is_registered() {
- local candidate="$1" entry
- for entry in "${REGISTERED[@]}"; do
- [[ "${entry}" == "${candidate}" ]] && return 0
- done
- return 1
-}
-# The scan proves nothing when it looks at nothing: a missing root only makes
-# `find` write to stderr, and an empty result would read as "all registered".
-# So every root must exist and the scan must find candidates.
-suite_roots=(codex-llm-proxy/test packages/llm-provider-attempt-contract)
-named_roots=(control-api mcp-host workflow-recipes host-context-controller control-ui)
-scan_roots_missing=0
-for scan_root in "${suite_roots[@]}" "${named_roots[@]}"; do
- if [[ ! -d "${ROOT}/${scan_root}" ]]; then
- fail "unlisted-suite scan root ${scan_root} is missing"
- scan_roots_missing=1
- fi
-done
-unlisted=0
-candidates=0
-while IFS= read -r rel; do
- [[ -n "${rel}" ]] || continue
- candidates=$((candidates + 1))
- if ! is_registered "${rel}"; then
- fail "unlisted Codex suite ${rel}"
- unlisted=1
- fi
-done < <(
- cd "${ROOT}" &&
- {
- find "${suite_roots[@]}" \
- -name node_modules -prune -o -type f \( -name '*.test.ts' -o -name '*.test.cjs' \) -print
- find "${named_roots[@]}" \
- \( -name node_modules -o -name dist -o -name .next -o -name coverage \) -prune -o \
- -type f -iname '*codex*' \( -name '*.test.ts' -o -name '*.test.tsx' \) -print
- } | sort -u
-)
-if [[ "${candidates}" -eq 0 ]]; then
- fail "unlisted-suite scan found no Codex test files"
-fi
-if [[ "${unlisted}" -eq 0 && "${scan_roots_missing}" -eq 0 && "${candidates}" -gt 0 ]]; then
- pass "every Codex suite is registered in T0 (${candidates} candidates scanned)"
+# Every candidate is registered in T0 or has an explicit CI/PG/runtime lane.
+# Discovery never certifies those other lanes; each requires its own physical receipt.
+discovery_registry="$(mktemp)"
+printf '%s\n' "${REGISTERED[@]}" >"${discovery_registry}"
+if ! node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" codex "${ROOT}" "${discovery_registry}"; then
+ FAIL=1
fi
+rm -f "${discovery_registry}"
-if [[ "${GROUPS_RUN}" -ne 12 ]]; then
- fail "expected all 12 T0 groups, ran ${GROUPS_RUN}"
+if [[ "${GROUPS_RUN}" -ne 13 ]]; then
+ fail "expected all 13 T0 groups, ran ${GROUPS_RUN}"
fi
if [[ "${FAIL}" -ne 0 ]]; then
diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.sh
index 2cb258393..a38db7e3b 100755
--- a/scripts/tests/test-grok-subscription-t0.sh
+++ b/scripts/tests/test-grok-subscription-t0.sh
@@ -15,10 +15,24 @@ set -u
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
FAIL=0
GROUPS_RUN=0
-EXPECTED_GROUPS=10
+EXPECTED_GROUPS=11
REGISTERED=()
COUNT_SUMMARY=""
+# Read-only inventory mode for hermetic discovery tests and operator inspection.
+# It does not enter package, cluster, Electron or browser execution.
+if [[ "${1:-}" == "--discovery-only" ]]; then
+ if [[ "$#" -ne 2 ]]; then
+ echo "usage: $0 --discovery-only <registered-file>" >&2
+ exit 2
+ fi
+ node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "$2"
+ exit "$?"
+elif [[ "$#" -ne 0 ]]; then
+ echo "unsupported T0 argument: $1" >&2
+ exit 2
+fi
+
pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }
@@ -166,6 +180,34 @@ run_group() {
rm -f "$log"
}
+run_node_group() {
+ local name="$1" rel log
+ shift
+ local files=()
+ for rel in "$@"; do
+ REGISTERED+=("${rel}")
+ require_file "$rel" || return 1
+ files+=("${ROOT}/$rel")
+ done
+ [[ ${#files[@]} -gt 0 ]] || { fail "$name: group listed no suite files"; return 1; }
+ log="$(mktemp)"
+ echo "── ${name} ──"
+ if ! node "$ROOT/scripts/tests/run-node-test-files.mjs" "${files[@]}" >"$log" 2>&1; then
+ fail "$name: command failed"
+ cat "$log"
+ rm -f "$log"
+ return 1
+ fi
+ if ! assert_executed_counts "$name" "$log" "${#files[@]}"; then
+ rm -f "$log"
+ return 1
+ fi
+ rm -f "$log"
+ GROUPS_RUN=$((GROUPS_RUN + 1))
+ pass "$name"
+}
+
+
require_ci_matrix_entry() {
local entry="$1"
if ! grep -Eq "^[[:space:]]+- ${entry}$" "${ROOT}/.github/workflows/ci-public.yml"; then
@@ -179,7 +221,6 @@ require_ci_matrix_entry() {
# proves they exist and that the CI real-PG lane asserts each one ran.
require_real_pg_suite() {
local rel="$1"
- REGISTERED+=("${rel}")
require_file "${rel}" || return 1
local suite
# The real-PG lane lists each suite by its file name, `.test.ts` included.
@@ -194,6 +235,10 @@ require_real_pg_suite() {
echo "Grok subscription T0 aggregator"
+run_node_group "subscription-discovery-and-offline-image-provider" \
+ "scripts/tests/subscription-t0-discovery.test.mjs" \
+ "scripts/e2e/fixtures/subscription-image-provider.test.mjs"
+
require_ci_matrix_entry "grok-llm-proxy"
require_ci_matrix_entry "packages/grok-provider-attempt-contract"
@@ -363,6 +408,10 @@ else
else
pass "desktop-app verify:electron"
run_group "desktop-app grok" "desktop-app" \
+ "test/subscriptionImageCollection.test.ts" \
+ "test/subscriptionImageChallenge.test.ts" \
+ "test/subscriptionImageRunContract.test.ts" \
+ "test/subscriptionAdmissionGuard.test.ts" \
"ui/src/constants/__tests__/attachments.test.ts" \
"ui/src/components/agents/__tests__/ComposerPanel.test.tsx"
fi
@@ -379,53 +428,14 @@ if require_file "scripts/tests/test-grok-llm-proxy-deploy-contract.sh" &&
fi
fi
-# A Grok suite that exists but is not listed above is lost coverage. Every
-# proxy/contract test file and every *grok* test file in the Grok-touching
-# packages must be registered in a group or as a real-PG presence check.
-is_registered() {
- local candidate="$1" entry
- for entry in "${REGISTERED[@]}"; do
- [[ "${entry}" == "${candidate}" ]] && return 0
- done
- return 1
-}
-# The scan proves nothing when it looks at nothing: a missing root only makes
-# `find` write to stderr, and an empty result would read as "all registered".
-# So every root must exist and the scan must find candidates.
-suite_roots=(grok-llm-proxy/test packages/grok-provider-attempt-contract)
-named_roots=(control-api mcp-host workflow-recipes host-context-controller control-ui)
-scan_roots_missing=0
-for scan_root in "${suite_roots[@]}" "${named_roots[@]}"; do
- if [[ ! -d "${ROOT}/${scan_root}" ]]; then
- fail "unlisted-suite scan root ${scan_root} is missing"
- scan_roots_missing=1
- fi
-done
-unlisted=0
-candidates=0
-while IFS= read -r rel; do
- [[ -n "${rel}" ]] || continue
- candidates=$((candidates + 1))
- if ! is_registered "${rel}"; then
- fail "unlisted Grok suite ${rel}"
- unlisted=1
- fi
-done < <(
- cd "${ROOT}" &&
- {
- find "${suite_roots[@]}" \
- -name node_modules -prune -o -type f \( -name '*.test.ts' -o -name '*.test.cjs' \) -print
- find "${named_roots[@]}" \
- \( -name node_modules -o -name dist -o -name .next -o -name coverage \) -prune -o \
- -type f -iname '*grok*' \( -name '*.test.ts' -o -name '*.test.tsx' \) -print
- } | sort -u
-)
-if [[ "${candidates}" -eq 0 ]]; then
- fail "unlisted-suite scan found no Grok test files"
-fi
-if [[ "${unlisted}" -eq 0 && "${scan_roots_missing}" -eq 0 && "${candidates}" -gt 0 ]]; then
- pass "every Grok suite is registered in T0 (${candidates} candidates scanned)"
+# Every candidate is registered in T0 or has an explicit CI/PG/runtime lane.
+# Discovery never certifies those other lanes; each requires its own physical receipt.
+discovery_registry="$(mktemp)"
+printf '%s\n' "${REGISTERED[@]}" >"${discovery_registry}"
+if ! node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "${discovery_registry}"; then
+ FAIL=1
fi
+rm -f "${discovery_registry}"
if [[ "${GROUPS_RUN}" -ne "${EXPECTED_GROUPS}" ]]; then
fail "expected all ${EXPECTED_GROUPS} T0 groups, ran ${GROUPS_RUN}"