Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(llm): verify complete subscription test discovery
Alfredo Lopez76bfa59b9224
diff --git a/scripts/tests/lib/subscription-t0-discovery.mjs b/scripts/tests/lib/subscription-t0-discovery.mjsnew file mode 100644index 000000000..d6f8a14ca--- /dev/null+++ b/scripts/tests/lib/subscription-t0-discovery.mjs@@ -0,0 +1,313 @@+import { lstatSync, readdirSync, readFileSync } from 'node:fs'+import path from 'node:path'+import { pathToFileURL } from 'node:url'++export const namedRoots = [+ 'control-api', 'mcp-host', 'workflow-recipes', 'host-context-controller',+ 'control-ui', 'desktop-app', 'tests/e2e', 'scripts/e2e',+]+const excludedDirectories = new Set([+ 'node_modules', 'dist', '.next', 'coverage', '.git', 'test-results',+ 'playwright-report', 'results', 'artifacts',+])+const testFile = /(?:\.test\.(?:ts|tsx|mjs|cjs)|\.spec\.(?:ts|tsx))$/i++function realPgLane(file) {+ return file.startsWith('control-api/test/') && file.endsWith('.realPostgres.integration.test.ts')+ ? 'real-pg:control-api'+ : undefined+}++export function isSubscriptionCandidate(file, provider) {+ const name = path.posix.basename(file).toLowerCase()+ const other = provider === 'codex' ? 'grok' : 'codex'+ return name.includes(provider) ||+ (!name.includes(other) && (name.includes('subscription') || name.includes('attachment')))+}++// These pre-existing shared suites have their own physical CI/PG consumers.+// New generic names do not inherit a lane: they must be registered explicitly.+const otherLaneSuites = new Map([+ ['control-api/test/subscriptionGrantIdentity.test.ts', 'general-ci:control-api'],+ ['control-api/test/config.subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],+ ['control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts', 'general-ci:control-api'],+ ['control-api/test/services.subscriptionCatalogSyncCron.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/db.codexSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.codexSubscriptionOAuth.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.grokProviderAttemptRedemption.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/pluginWorkloadSdkGrokDualLedger.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts', 'real-pg:control-api'],+ ['mcp-host/src/__tests__/server.attachments.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/agent/__tests__/attachmentRead.sqlite.integration.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/agent/__tests__/attachmentRead.integration.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/agent/__tests__/incomingAttachments.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/runtime/__tests__/fileAttachmentDelivery.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/core/orchestration/__tests__/toolUseLoopSingleTool.attachmentRead.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/core/tools/__tests__/nativeToolRegistry.attachmentRead.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/core/tools/__tests__/generatedArtifactAttachments.test.ts', 'general-ci:mcp-host'],+ ['mcp-host/src/core/tools/__tests__/attachmentRead.test.ts', 'general-ci:mcp-host'],+ ['desktop-app/ui/src/lib/__tests__/chatMessageAttachments.test.ts', 'general-ci:desktop-app'],+])+const runtimeSuites = new Map([+ ['desktop-app/test/e2e-playwright/subscription-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts'],+ ['desktop-app/test/e2e-playwright/codex-image-input.spec.ts', 'desktop-app/test/e2e-playwright/playwright.codex-image.config.ts'],+ ['desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts', 'desktop-app/test/e2e-playwright/playwright.config.ts'],+ ['tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts', 'tests/e2e/playwright/playwright.subscription-admission.config.ts'],+ ['tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts', 'tests/e2e/playwright/playwright.codex-approved-tools.config.ts'],+ ['tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts', 'tests/e2e/playwright/playwright.codex-subscription.config.ts'],+ ['scripts/e2e/e2e-codex-subscription-runtime.sh', 'scripts/e2e/e2e-codex-subscription-runtime.sh'],+ ['scripts/e2e/e2e-codex-subscription-playwright.sh', 'scripts/e2e/e2e-codex-subscription-playwright.sh'],+ ['scripts/e2e/e2e-codex-subscription-network-boundary.sh', 'scripts/e2e/e2e-codex-subscription-network-boundary.sh'],+])++function regularFile(root, file) {+ try { return lstatSync(path.join(root, file)).isFile() } catch { return false }+}+function walk(root, directory) {+ const files = []+ for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) {+ const relative = `${directory}/${entry.name}`+ if (entry.isDirectory() && !excludedDirectories.has(entry.name)) files.push(...walk(root, relative))+ else if (entry.isFile()) files.push(relative)+ }+ return files+}+function isTest(file) {+ return testFile.test(file) || /^e2e-.*\.sh$/i.test(path.posix.basename(file))+}+function configPatterns(source, key) {+ const block = new RegExp(`${key}:\\s*\\[([\\s\\S]*?)\\]`).exec(source)?.[1] ?? ''+ return [...block.matchAll(/['"]([^'"]+)['"]/g)].map(match => match[1])+}++function configSelectors(source, key) {+ const values = []+ const assignments = new RegExp(+ `${key}:\\s*(\\[[^\\]]*\\]|/(?:\\\\.|[^/\\\\])+/[a-z]*|'[^']*'|"[^"]*")`,+ 'g',+ )+ for (const match of source.matchAll(assignments)) {+ const value = match[1]+ if (value.startsWith('[')) values.push(...configPatterns(`${key}: [${value.slice(1, -1)}]`, key))+ else if (value.startsWith('/')) values.push(new RegExp(value.slice(1, value.lastIndexOf('/'))))+ else values.push(value.slice(1, -1))+ }+ return values+}++function matchesPlaywrightSelector(relative, selector) {+ if (selector instanceof RegExp) return selector.test(relative)+ return path.matchesGlob(relative, selector) ||+ (!selector.includes('/') && path.matchesGlob(path.posix.basename(relative), selector))+}++function projectBlocks(source) {+ const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)+ if (!projectsAssignment) return []+ const arrayStart = source.indexOf('[', projectsAssignment.index)+ const blocks = []+ let depth = 0+ let blockDepth = 0+ let blockStart = -1+ let quote+ for (let index = arrayStart; index < source.length; index += 1) {+ const character = source[index]+ if (quote) {+ if (character === '\\') index += 1+ else if (character === quote) quote = undefined+ continue+ }+ if (character === '\'' || character === '"' || character === '`') {+ quote = character+ continue+ }+ if (character === '[' || character === '{') {+ depth += 1+ if (character === '{' && depth === 2) {+ blockDepth = depth+ blockStart = index+ }+ continue+ }+ if (character === ']' || character === '}') {+ depth -= 1+ if (character === ']' && depth === 0) return blocks+ }+ if (character === '}') {+ if (blockStart >= 0 && depth === 1) {+ blocks.push(source.slice(blockStart, index + 1))+ blockStart = -1+ }+ }+ }+ return blocks+}++function playwrightCollectsFile(source, configDirectory, file) {+ if (!source.includes('defineConfig')) return false+ const relative = path.posix.relative(configDirectory, file)+ const testDirectory = /(?:^|[{,\n]\s*)testDir:\s*['"](.+)['"]/.exec(source)?.[1]+ if (testDirectory) {+ const normalized = path.posix.normalize(testDirectory)+ if (normalized !== '.' && !relative.startsWith(`${normalized}/`)) return false+ }++ const projectsAssignment = /(?:^|\n)\s*projects:\s*\[/.exec(source)+ const topLevelSource = projectsAssignment+ ? source.slice(0, projectsAssignment.index)+ : source+ const ignored = configSelectors(topLevelSource, 'testIgnore')+ .some(selector => matchesPlaywrightSelector(relative, selector))+ if (ignored) return false++ const topLevelSelectors = configSelectors(topLevelSource, 'testMatch')+ if (topLevelSelectors.length > 0) {+ return topLevelSelectors.some(selector => matchesPlaywrightSelector(relative, selector))+ }++ const blocks = projectBlocks(source)+ if (blocks.length === 0) return true+ return blocks.some(block => {+ const blockIgnored = configSelectors(block, 'testIgnore')+ .some(selector => matchesPlaywrightSelector(relative, selector))+ const selectors = configSelectors(block, 'testMatch')+ if (selectors.length === 0) return !blockIgnored+ return !blockIgnored &&+ selectors.some(selector => matchesPlaywrightSelector(relative, selector))+ })+}++export function verifyRuntimeConsumer(root, file, consumer) {+ if (!regularFile(root, file) || !regularFile(root, consumer)) return undefined++ if (/\.sh$/i.test(consumer)) {+ const source = readFileSync(path.join(root, consumer), 'utf8')+ const executable = (lstatSync(path.join(root, consumer)).mode & 0o111) !== 0+ if (consumer !== file || !source.startsWith('#!') || !executable) return undefined+ return `${consumer} (explicit self entrypoint; separate physical receipt required)`+ }++ const source = readFileSync(path.join(root, consumer), 'utf8')+ if (!playwrightCollectsFile(source, path.posix.dirname(consumer), file)) return undefined+ return `${consumer} (collection-verified registry entry)`+}++function verifyOtherConsumer(root, file, lane) {+ const service = lane.split(':')[1]+ const ciPath = '.github/workflows/ci-public.yml'+ const configPath = `${service}/vitest.config.ts`+ if (!regularFile(root, ciPath) || !regularFile(root, configPath)) return undefined+ const ci = readFileSync(path.join(root, ciPath), 'utf8')+ const config = readFileSync(path.join(root, configPath), 'utf8')+ const relative = file.slice(service.length + 1)+ const included = configPatterns(config, 'include').some(pattern => path.matchesGlob(relative, pattern))+ const excluded = configPatterns(config, 'exclude').some(pattern => path.matchesGlob(relative, pattern))+ if (!included || excluded) return undefined+ if (lane.startsWith('real-pg:')) {+ const pgJob = ci.split(' control-api-migration:')[1]?.split(/\n [a-z][a-z0-9-]*:/)[0]+ if (+ !pgJob ||+ !/CONTROL_API_REAL_PG_REQUIRED:\s*'1'/.test(pgJob) ||+ !/npm test.*(?:--run )?realPostgres/.test(pgJob)+ ) return undefined+ return `${ciPath}#control-api-migration (realPostgres filter); ${configPath}`+ }+ if (!new RegExp(`^\\s*- ${service}\\s*$`, 'm').test(ci) || !ci.includes('npm test')) return undefined+ return `${ciPath}#test (${service}); ${configPath}`+}++export function auditSubscriptionDiscovery(provider, root, registered) {+ if (!['grok', 'codex'].includes(provider)) throw new Error('provider must be grok or codex')+ const suiteRoots = provider === 'grok'+ ? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']+ : ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']+ const violations = [], candidates = [], roots = [...suiteRoots, ...namedRoots]+ const filesByRoot = new Map()+ const registeredSet = new Set(registered)+ if (registeredSet.size !== registered.length) violations.push('duplicate registered physical suite')+ for (const file of registered) {+ if (path.posix.isAbsolute(file) || path.posix.normalize(file) !== file || file.startsWith('../') || !regularFile(root, file)) {+ violations.push(`missing or invalid registered suite ${file}`)+ }+ }+ for (const directory of roots) {+ let valid = false+ try { valid = lstatSync(path.join(root, directory)).isDirectory() } catch { /* Missing is a violation. */ }+ if (!valid) { violations.push(`missing discovery root ${directory}`); continue }+ const files = walk(root, directory).filter(isTest)+ if (files.length === 0) violations.push(`empty discovery root ${directory}`)+ filesByRoot.set(directory, files)+ }+ for (const [directory, files] of filesByRoot) {+ for (const file of files) {+ if (!suiteRoots.includes(directory) && !isSubscriptionCandidate(file, provider)) continue+ let lane, consumer+ const pgLane = realPgLane(file)+ const pgRegistered = Boolean(pgLane) &&+ registeredSet.has(file) && otherLaneSuites.get(file) === pgLane+ if (runtimeSuites.has(file)) {+ lane = 'runtime-opt-in'+ if (registeredSet.has(file)) {+ violations.push(`runtime suite cannot be registered as a T0 unit ${file}`)+ continue+ }+ consumer = verifyRuntimeConsumer(root, file, runtimeSuites.get(file))+ if (!consumer) violations.push(`runtime consumer does not collect ${file}`)+ } else if (pgRegistered) {+ lane = pgLane+ consumer = verifyOtherConsumer(root, file, pgLane)+ if (!consumer) violations.push(`unverified ${pgLane} consumer for ${file}`)+ else violations.push(`real-PG suite cannot be registered as a T0 unit ${file}`)+ } else if (registeredSet.has(file)) {+ lane = 'T0'+ consumer = 'physical T0 group; separate producer/reporter required'+ } else if (pgLane && otherLaneSuites.get(file) === pgLane) {+ lane = pgLane+ consumer = verifyOtherConsumer(root, file, lane)+ if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)+ } else if (otherLaneSuites.has(file)) {+ lane = otherLaneSuites.get(file)+ consumer = verifyOtherConsumer(root, file, lane)+ if (!consumer) violations.push(`unverified ${lane} consumer for ${file}`)+ } else {+ violations.push(`unlisted ${provider} subscription suite ${file}`)+ lane = 'UNLISTED'+ }+ candidates.push({ file, lane, consumer, evidence: 'NOT_RUN_HERE', requiredPhysicalEvidence: true })+ }+ }+ if (candidates.length === 0) violations.push(`empty ${provider} subscription candidate scan`)+ return { provider, roots, candidates: candidates.sort((a, b) => a.file.localeCompare(b.file)), violations }+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [provider, root, registryFile, outputMode] = process.argv.slice(2)+ if (!provider || !root || !registryFile || (outputMode && outputMode !== '--json')) {+ console.error('usage: subscription-t0-discovery.mjs <grok|codex> <root> <registered-file> [--json]')+ process.exit(2)+ }+ const registered = readFileSync(registryFile, 'utf8').split(/\r?\n/).filter(Boolean)+ const result = auditSubscriptionDiscovery(provider, root, registered)+ if (outputMode === '--json') console.log(JSON.stringify(result))+ else {+ for (const entry of result.candidates.filter(entry => entry.lane !== 'T0')) {+ console.log(`INVENTORY ${entry.lane} NOT_RUN_HERE: ${entry.file}; physical evidence required via ${entry.consumer ?? 'explicit registration'}`)+ }+ for (const violation of result.violations) console.error(`FAIL: ${violation}`)+ console.log(`Discovery inventory: ${provider}, ${result.candidates.length} candidates, ${result.violations.length} violations; execution evidence is separate`)+ }+ process.exit(result.violations.length ? 1 : 0)+}diff --git a/scripts/tests/subscription-t0-discovery.test.mjs b/scripts/tests/subscription-t0-discovery.test.mjsnew file mode 100644index 000000000..313199621--- /dev/null+++ b/scripts/tests/subscription-t0-discovery.test.mjs@@ -0,0 +1,311 @@+import assert from 'node:assert/strict'+import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import path from 'node:path'+import test from 'node:test'+import {+ auditSubscriptionDiscovery,+ isSubscriptionCandidate,+ namedRoots,+ verifyRuntimeConsumer,+} from './lib/subscription-t0-discovery.mjs'++function suiteRoots(provider) {+ return provider === 'grok'+ ? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']+ : ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']+}++function createFile(root, relative, content = '') {+ const target = path.join(root, relative)+ mkdirSync(path.dirname(target), { recursive: true })+ writeFileSync(target, content)+ return relative+}++function makeFixture(t, provider = 'codex') {+ const root = mkdtempSync(path.join(tmpdir(), `subscription-discovery-${provider}-`))+ t.after(() => rmSync(root, { recursive: true, force: true }))+ const registered = []+ for (const directory of [...suiteRoots(provider), ...namedRoots]) {+ mkdirSync(path.join(root, directory), { recursive: true })+ }+ for (const [index, directory] of suiteRoots(provider).entries()) {+ const file = `${directory}/${provider}-suite-${index}.test.ts`+ createFile(root, file)+ registered.push(file)+ }+ namedRoots.forEach((directory, index) => {+ createFile(root, `${directory}/baseline-${index}.test.ts`)+ })+ return { root, registered }+}++test('admits the three formerly omitted roots through their physical file entries', t => {+ const fixture = makeFixture(t)+ const desktop = createFile(+ fixture.root,+ 'desktop-app/ui/src/constants/__tests__/attachments.test.ts',+ )+ const host = createFile(+ fixture.root,+ 'mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts',+ )+ const e2e = createFile(+ fixture.root,+ 'tests/e2e/integration/codex-subscription-contract-freeze.test.ts',+ )+ const scripts = createFile(+ fixture.root,+ 'scripts/e2e/prepare-codex-approved-tools.test.mjs',+ )+ const registered = [...fixture.registered, desktop, host, e2e, scripts]+ const result = auditSubscriptionDiscovery('codex', fixture.root, registered)++ assert.deepEqual(result.violations, [])+ const lanes = new Map(result.candidates.map(entry => [entry.file, entry.lane]))+ for (const file of [desktop, host, e2e, scripts]) assert.equal(lanes.get(file), 'T0')+ for (const root of ['desktop-app', 'tests/e2e', 'scripts/e2e']) {+ assert.ok(result.roots.includes(root))+ }+})++test('generic attachment candidates belong to both provider lanes; provider names stay isolated', () => {+ const attachment = 'any/deep/attachments.test.ts'+ const codex = 'mcp-host/src/llm/__tests__/codexSubscription.test.ts'+ const grok = 'mcp-host/src/llm/__tests__/grokSubscription.test.ts'+ assert.equal(isSubscriptionCandidate(attachment, 'codex'), true)+ assert.equal(isSubscriptionCandidate(attachment, 'grok'), true)+ assert.equal(isSubscriptionCandidate(codex, 'codex'), true)+ assert.equal(isSubscriptionCandidate(codex, 'grok'), false)+ assert.equal(isSubscriptionCandidate(grok, 'grok'), true)+ assert.equal(isSubscriptionCandidate(grok, 'codex'), false)+})++test('missing and empty discovery roots fail with their exact causal violation', t => {+ const missing = makeFixture(t)+ rmSync(path.join(missing.root, 'tests/e2e'), { recursive: true, force: true })+ const missingResult = auditSubscriptionDiscovery('codex', missing.root, missing.registered)+ assert.ok(missingResult.violations.includes('missing discovery root tests/e2e'))++ const empty = makeFixture(t, 'grok')+ rmSync(path.join(empty.root, 'mcp-host/baseline-1.test.ts'))+ const emptyResult = auditSubscriptionDiscovery('grok', empty.root, empty.registered)+ assert.ok(emptyResult.violations.includes('empty discovery root mcp-host'))+})++test('duplicate and missing registered physical suites fail closed', t => {+ const fixture = makeFixture(t)+ const duplicateResult = auditSubscriptionDiscovery(+ 'codex',+ fixture.root,+ [...fixture.registered, fixture.registered[0]],+ )+ assert.ok(duplicateResult.violations.includes('duplicate registered physical suite'))++ const missingRelative = 'control-api/test/missing-registered-suite.test.ts'+ createFile(path.dirname(path.join(fixture.root, missingRelative)), path.basename(missingRelative))+ rmSync(path.join(fixture.root, missingRelative))+ const missingResult = auditSubscriptionDiscovery(+ 'codex',+ fixture.root,+ [...fixture.registered, missingRelative],+ )+ assert.ok(+ missingResult.violations.includes(`missing or invalid registered suite ${missingRelative}`),+ )+})++test('an unlisted candidate is UNLISTED even when another candidate is registered', t => {+ const fixture = makeFixture(t)+ const unlisted = createFile(+ fixture.root,+ 'mcp-host/src/llm/__tests__/newCodexSurface.test.ts',+ )+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.ok(result.violations.includes(`unlisted codex subscription suite ${unlisted}`))+ const entry = result.candidates.find(candidate => candidate.file === unlisted)+ assert.equal(entry?.lane, 'UNLISTED')+})++test('registered real-PG suites are never confused with executed T0 suites', t => {+ const fixture = makeFixture(t)+ const suite = 'control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts'+ createFile(fixture.root, suite)+ createFile(+ fixture.root,+ 'control-api/vitest.config.ts',+ "export default { include: ['test/**/*.test.ts'] }\n",+ )+ createFile(+ fixture.root,+ '.github/workflows/ci-public.yml',+ ` control-api-migration:+ steps:+ - run: |+ CONTROL_API_REAL_PG_REQUIRED: '1'+ npm test -- --run realPostgres+`,+ )++ const registeredResult = auditSubscriptionDiscovery(+ 'codex',+ fixture.root,+ [...fixture.registered, suite],+ )+ const registeredEntry = registeredResult.candidates.find(entry => entry.file === suite)+ assert.equal(registeredEntry?.lane, 'real-pg:control-api')+ assert.ok(+ registeredResult.violations.includes(`real-PG suite cannot be registered as a T0 unit ${suite}`),+ )++ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.deepEqual(result.violations, [])+ const entry = result.candidates.find(candidate => candidate.file === suite)+ assert.equal(entry?.lane, 'real-pg:control-api')+ assert.match(String(entry?.consumer), /control-api-migration/)+})++test('a realPostgres suffix alone is not a silent other-lane registry', t => {+ const fixture = makeFixture(t)+ const unknown = createFile(+ fixture.root,+ 'control-api/test/unknownSubscription.realPostgres.integration.test.ts',+ )+ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.ok(result.violations.includes(`unlisted codex subscription suite ${unknown}`))+})++test('general CI lanes verify both matrix ownership and config collection', t => {+ const fixture = makeFixture(t)+ const suite = 'control-api/test/subscriptionGrantIdentity.test.ts'+ const config = 'control-api/vitest.config.ts'+ const workflow = '.github/workflows/ci-public.yml'+ createFile(fixture.root, suite)+ createFile(fixture.root, workflow, `jobs:+ test:+ strategy:+ matrix:+ service:+ - control-api+ steps:+ - run: npm test+`)+ createFile(fixture.root, config, "export default { include: ['test/**/*.test.ts'] }\n")++ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.deepEqual(result.violations, [])+ const entry = result.candidates.find(candidate => candidate.file === suite)+ assert.equal(entry?.lane, 'general-ci:control-api')+ assert.match(String(entry?.consumer), /#test \(control-api\)/)++ writeFileSync(path.join(fixture.root, config), "export default { include: ['test/other.test.ts'] }\n")+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.ok(negative.violations.includes(`unverified general-ci:control-api consumer for ${suite}`))+})++test('Playwright runtime lanes require the registered config to collect the exact spec', t => {+ const fixture = makeFixture(t)+ const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'+ const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'+ createFile(fixture.root, spec)+ createFile(+ fixture.root,+ config,+ `import { defineConfig } from '@playwright/test'+export default defineConfig({+ testDir: '.',+ testMatch: 'control-ui/codex-subscription-admission.spec.ts',+})+`,+ )++ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.deepEqual(result.violations, [])+ const entry = result.candidates.find(candidate => candidate.file === spec)+ assert.equal(entry?.lane, 'runtime-opt-in')+ assert.match(String(entry?.consumer), /collection-verified registry entry/)++ writeFileSync(+ path.join(fixture.root, config),+ `import { defineConfig } from '@playwright/test'+export default defineConfig({+ testDir: '.',+ testMatch: 'control-ui/other.spec.ts',+})+`,+ )+ assert.equal(verifyRuntimeConsumer(fixture.root, spec, config), undefined)+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.ok(negative.violations.includes(`runtime consumer does not collect ${spec}`))+})++test('shell runtime entries are explicit self entrypoints and reject non-entrypoint files', t => {+ const fixture = makeFixture(t)+ const script = 'scripts/e2e/e2e-codex-subscription-runtime.sh'+ createFile(fixture.root, script, '#!/usr/bin/env bash\nset -euo pipefail\n')+ chmodSync(path.join(fixture.root, script), 0o755)++ const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.deepEqual(result.violations, [])+ const entry = result.candidates.find(candidate => candidate.file === script)+ assert.equal(entry?.lane, 'runtime-opt-in')+ assert.match(String(entry?.consumer), /explicit self entrypoint/)++ chmodSync(path.join(fixture.root, script), 0o644)+ const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)+ assert.ok(negative.violations.includes(`runtime consumer does not collect ${script}`))+})++test('a Playwright default project can collect a spec outside another project selector', t => {+ const fixture = makeFixture(t)+ const spec = 'desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts'+ const config = 'desktop-app/test/e2e-playwright/playwright.config.ts'+ createFile(fixture.root, spec)+ createFile(+ fixture.root,+ config,+ `import { defineConfig } from '@playwright/test'+export default defineConfig({+ testDir: '.',+ projects: [+ {+ name: 'default',+ testIgnore: ['**/other-lane.spec.ts'],+ },+ {+ name: 'special',+ testMatch: /other-lane\\.spec\\.ts/,+ },+ ],+})+`,+ )++ assert.ok(verifyRuntimeConsumer(fixture.root, spec, config))+})++test('runtime suites cannot enter the T0 registry', t => {+ const fixture = makeFixture(t)+ const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'+ const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'+ createFile(fixture.root, spec)+ createFile(+ fixture.root,+ config,+ `import { defineConfig } from '@playwright/test'+export default defineConfig({+ testDir: '.',+ testMatch: 'control-ui/codex-subscription-admission.spec.ts',+})+`,+ )+ const result = auditSubscriptionDiscovery(+ 'codex',+ fixture.root,+ [...fixture.registered, spec],+ )+ assert.ok(+ result.violations.includes(`runtime suite cannot be registered as a T0 unit ${spec}`),+ )+})diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 1a6b2d4a7..289c00547 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -14,6 +14,20 @@ FAIL=0GROUPS_RUN=0REGISTERED=()+# Read-only inventory mode for hermetic discovery tests and operator inspection.+# It does not enter package, cluster, Electron or browser execution.+if [[ "${1:-}" == "--discovery-only" ]]; then+ if [[ "$#" -ne 2 ]]; then+ echo "usage: $0 --discovery-only <registered-file>" >&2+ exit 2+ fi+ node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" codex "${ROOT}" "$2"+ exit "$?"+elif [[ "$#" -ne 0 ]]; then+ echo "unsupported T0 argument: $1" >&2+ exit 2+fi+pass() { echo "PASS: $1"; }fail() { echo "FAIL: $1"; FAIL=1; }@@ -180,7 +194,6 @@ require_ci_matrix_entry() {# proves they exist and that the CI real-PG lane asserts each one ran.require_real_pg_suite() {local rel="$1"- REGISTERED+=("${rel}")require_file "${rel}" || return 1local suite# The real-PG lane lists each suite by its file name, `.test.ts` included.@@ -221,7 +234,12 @@ firun_group "shared-contract" "packages/llm-provider-attempt-contract" "index.test.cjs"run_group "codex-catalog-projection" "packages/codex-catalog-projection" "index.test.cjs"+# This is a filesystem-only freeze contract, not a browser/runtime journey.+run_group "codex-upstream-contract-freeze" "tests/e2e" "integration/codex-subscription-contract-freeze.test.ts"+run_node_group "approved-tools-fixtures-and-runner" \+ "scripts/tests/subscription-t0-discovery.test.mjs" \+ "scripts/e2e/fixtures/subscription-image-provider.test.mjs" \"scripts/tests/run-node-test-files.test.mjs" \"tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs" \"scripts/e2e/prepare-codex-approved-tools.test.mjs" \@@ -310,6 +328,7 @@ run_group "codex-llm-proxy" "codex-llm-proxy" \run_group "mcp-host" "mcp-host" \"src/__tests__/bodylimits.test.ts" \+ "src/llm/__tests__/attachmentBudgetRefusal.test.ts" \"src/llm/__tests__/imageSource.test.ts" \"src/capabilities/toolCatalogTools.test.ts" \"src/core/orchestration/__tests__/approvedToolsLifecycle.integration.test.ts" \@@ -399,62 +418,28 @@ else"src/__tests__/devIsolation.test.ts" \"ui/src/components/agents/__tests__/ComposerPanel.test.tsx" \"ui/src/components/agents/__tests__/ModelSelector.test.tsx" \+ "test/subscriptionImageCollection.test.ts" \+ "test/subscriptionImageChallenge.test.ts" \+ "test/subscriptionImageRunContract.test.ts" \+ "test/subscriptionAdmissionGuard.test.ts" \+ "test/codexImageChallenge.test.ts" \"ui/src/constants/__tests__/attachments.test.ts" \"ui/src/hooks/__tests__/useHostModels.test.tsx" \"ui/src/hooks/domain/__tests__/useAgentChatController.pendingModel.test.tsx"fifi-# A Codex suite that exists but is not listed above is lost coverage. Every-# proxy/contract test file and every *codex* test file in the Codex-touching-# packages must be registered in a group or as a real-PG presence check.-is_registered() {- local candidate="$1" entry- for entry in "${REGISTERED[@]}"; do- [[ "${entry}" == "${candidate}" ]] && return 0- done- return 1-}-# The scan proves nothing when it looks at nothing: a missing root only makes-# `find` write to stderr, and an empty result would read as "all registered".-# So every root must exist and the scan must find candidates.-suite_roots=(codex-llm-proxy/test packages/llm-provider-attempt-contract)-named_roots=(control-api mcp-host workflow-recipes host-context-controller control-ui)-scan_roots_missing=0-for scan_root in "${suite_roots[@]}" "${named_roots[@]}"; do- if [[ ! -d "${ROOT}/${scan_root}" ]]; then- fail "unlisted-suite scan root ${scan_root} is missing"- scan_roots_missing=1- fi-done-unlisted=0-candidates=0-while IFS= read -r rel; do- [[ -n "${rel}" ]] || continue- candidates=$((candidates + 1))- if ! is_registered "${rel}"; then- fail "unlisted Codex suite ${rel}"- unlisted=1- fi-done < <(- cd "${ROOT}" &&- {- find "${suite_roots[@]}" \- -name node_modules -prune -o -type f \( -name '*.test.ts' -o -name '*.test.cjs' \) -print- find "${named_roots[@]}" \- \( -name node_modules -o -name dist -o -name .next -o -name coverage \) -prune -o \- -type f -iname '*codex*' \( -name '*.test.ts' -o -name '*.test.tsx' \) -print- } | sort -u-)-if [[ "${candidates}" -eq 0 ]]; then- fail "unlisted-suite scan found no Codex test files"-fi-if [[ "${unlisted}" -eq 0 && "${scan_roots_missing}" -eq 0 && "${candidates}" -gt 0 ]]; then- pass "every Codex suite is registered in T0 (${candidates} candidates scanned)"+# Every candidate is registered in T0 or has an explicit CI/PG/runtime lane.+# Discovery never certifies those other lanes; each requires its own physical receipt.+discovery_registry="$(mktemp)"+printf '%s\n' "${REGISTERED[@]}" >"${discovery_registry}"+if ! node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" codex "${ROOT}" "${discovery_registry}"; then+ FAIL=1fi+rm -f "${discovery_registry}"-if [[ "${GROUPS_RUN}" -ne 12 ]]; then- fail "expected all 12 T0 groups, ran ${GROUPS_RUN}"+if [[ "${GROUPS_RUN}" -ne 13 ]]; then+ fail "expected all 13 T0 groups, ran ${GROUPS_RUN}"fiif [[ "${FAIL}" -ne 0 ]]; thendiff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.shindex 2cb258393..a38db7e3b 100755--- a/scripts/tests/test-grok-subscription-t0.sh+++ b/scripts/tests/test-grok-subscription-t0.sh@@ -15,10 +15,24 @@ set -uROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"FAIL=0GROUPS_RUN=0-EXPECTED_GROUPS=10+EXPECTED_GROUPS=11REGISTERED=()COUNT_SUMMARY=""+# Read-only inventory mode for hermetic discovery tests and operator inspection.+# It does not enter package, cluster, Electron or browser execution.+if [[ "${1:-}" == "--discovery-only" ]]; then+ if [[ "$#" -ne 2 ]]; then+ echo "usage: $0 --discovery-only <registered-file>" >&2+ exit 2+ fi+ node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "$2"+ exit "$?"+elif [[ "$#" -ne 0 ]]; then+ echo "unsupported T0 argument: $1" >&2+ exit 2+fi+pass() { echo "PASS: $1"; }fail() { echo "FAIL: $1"; FAIL=1; }@@ -166,6 +180,34 @@ run_group() {rm -f "$log"}+run_node_group() {+ local name="$1" rel log+ shift+ local files=()+ for rel in "$@"; do+ REGISTERED+=("${rel}")+ require_file "$rel" || return 1+ files+=("${ROOT}/$rel")+ done+ [[ ${#files[@]} -gt 0 ]] || { fail "$name: group listed no suite files"; return 1; }+ log="$(mktemp)"+ echo "ββ ${name} ββ"+ if ! node "$ROOT/scripts/tests/run-node-test-files.mjs" "${files[@]}" >"$log" 2>&1; then+ fail "$name: command failed"+ cat "$log"+ rm -f "$log"+ return 1+ fi+ if ! assert_executed_counts "$name" "$log" "${#files[@]}"; then+ rm -f "$log"+ return 1+ fi+ rm -f "$log"+ GROUPS_RUN=$((GROUPS_RUN + 1))+ pass "$name"+}++require_ci_matrix_entry() {local entry="$1"if ! grep -Eq "^[[:space:]]+- ${entry}$" "${ROOT}/.github/workflows/ci-public.yml"; then@@ -179,7 +221,6 @@ require_ci_matrix_entry() {# proves they exist and that the CI real-PG lane asserts each one ran.require_real_pg_suite() {local rel="$1"- REGISTERED+=("${rel}")require_file "${rel}" || return 1local suite# The real-PG lane lists each suite by its file name, `.test.ts` included.@@ -194,6 +235,10 @@ require_real_pg_suite() {echo "Grok subscription T0 aggregator"+run_node_group "subscription-discovery-and-offline-image-provider" \+ "scripts/tests/subscription-t0-discovery.test.mjs" \+ "scripts/e2e/fixtures/subscription-image-provider.test.mjs"+require_ci_matrix_entry "grok-llm-proxy"require_ci_matrix_entry "packages/grok-provider-attempt-contract"@@ -363,6 +408,10 @@ elseelsepass "desktop-app verify:electron"run_group "desktop-app grok" "desktop-app" \+ "test/subscriptionImageCollection.test.ts" \+ "test/subscriptionImageChallenge.test.ts" \+ "test/subscriptionImageRunContract.test.ts" \+ "test/subscriptionAdmissionGuard.test.ts" \"ui/src/constants/__tests__/attachments.test.ts" \"ui/src/components/agents/__tests__/ComposerPanel.test.tsx"fi@@ -379,53 +428,14 @@ if require_file "scripts/tests/test-grok-llm-proxy-deploy-contract.sh" &&fifi-# A Grok suite that exists but is not listed above is lost coverage. Every-# proxy/contract test file and every *grok* test file in the Grok-touching-# packages must be registered in a group or as a real-PG presence check.-is_registered() {- local candidate="$1" entry- for entry in "${REGISTERED[@]}"; do- [[ "${entry}" == "${candidate}" ]] && return 0- done- return 1-}-# The scan proves nothing when it looks at nothing: a missing root only makes-# `find` write to stderr, and an empty result would read as "all registered".-# So every root must exist and the scan must find candidates.-suite_roots=(grok-llm-proxy/test packages/grok-provider-attempt-contract)-named_roots=(control-api mcp-host workflow-recipes host-context-controller control-ui)-scan_roots_missing=0-for scan_root in "${suite_roots[@]}" "${named_roots[@]}"; do- if [[ ! -d "${ROOT}/${scan_root}" ]]; then- fail "unlisted-suite scan root ${scan_root} is missing"- scan_roots_missing=1- fi-done-unlisted=0-candidates=0-while IFS= read -r rel; do- [[ -n "${rel}" ]] || continue- candidates=$((candidates + 1))- if ! is_registered "${rel}"; then- fail "unlisted Grok suite ${rel}"- unlisted=1- fi-done < <(- cd "${ROOT}" &&- {- find "${suite_roots[@]}" \- -name node_modules -prune -o -type f \( -name '*.test.ts' -o -name '*.test.cjs' \) -print- find "${named_roots[@]}" \- \( -name node_modules -o -name dist -o -name .next -o -name coverage \) -prune -o \- -type f -iname '*grok*' \( -name '*.test.ts' -o -name '*.test.tsx' \) -print- } | sort -u-)-if [[ "${candidates}" -eq 0 ]]; then- fail "unlisted-suite scan found no Grok test files"-fi-if [[ "${unlisted}" -eq 0 && "${scan_roots_missing}" -eq 0 && "${candidates}" -gt 0 ]]; then- pass "every Grok suite is registered in T0 (${candidates} candidates scanned)"+# Every candidate is registered in T0 or has an explicit CI/PG/runtime lane.+# Discovery never certifies those other lanes; each requires its own physical receipt.+discovery_registry="$(mktemp)"+printf '%s\n' "${REGISTERED[@]}" >"${discovery_registry}"+if ! node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "${discovery_registry}"; then+ FAIL=1fi+rm -f "${discovery_registry}"if [[ "${GROUPS_RUN}" -ne "${EXPECTED_GROUPS}" ]]; thenfail "expected all ${EXPECTED_GROUPS} T0 groups, ran ${GROUPS_RUN}"