Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(subscriptions): gate every directly emitted proxy code against the fixture taxonomy (#739)

Alfredo Lopezcbd6bbb3d3a5
diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
index 542460ddc..4e43e78a1 100644
--- a/codex-llm-proxy/test/streamLimitsFreeze.test.ts
+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
@@ -65,6 +65,47 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; construct
return { codes, sites, constructions }
}
+// Every direct HTTP refusal in src: `reject(res, <status>, <code>)`.
+const REJECT_SITE = /reject\(res,/g
+// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.
+const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/
+// The only computed codes a refusal may carry, as their exact source text.
+// `err.code` is a RequestLimitError (always provider_unavailable) and
+// `mapped.code` is a transport or control-api code passed through mapError,
+// whose transport half the scanner above already gates. Any other computed
+// form cannot be checked against the taxonomy, so it fails here.
+const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const
+
+function emittedRejectCodes(): {
+ codes: Set<string>
+ computed: Set<string>
+ sites: number
+ occurrences: number
+} {
+ const codes = new Set<string>()
+ const computed = new Set<string>()
+ let sites = 0
+ let occurrences = 0
+ for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
+ const source = readFileSync(new URL(name, srcDir), 'utf8')
+ occurrences += source.split('reject(res,').length - 1
+ for (const match of source.matchAll(REJECT_SITE)) {
+ const args = callArguments(source, match.index! + 'reject'.length)
+ const code = args[2] ?? ''
+ const literal = REJECT_CODE_LITERAL.exec(code)
+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
+ expect(
+ literal !== null || allowed,
+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
+ ).toBe(true)
+ if (literal) codes.add(literal[1]!)
+ else computed.add(code)
+ sites += 1
+ }
+ }
+ return { codes, computed, sites, occurrences }
+}
+
describe('codex-subscription stream limits freeze', () => {
it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
// StreamGate and the upstream deadlines enforce these published bounds
@@ -106,4 +147,21 @@ describe('codex-subscription stream limits freeze', () => {
const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()
expect(unpublished, 'emitted transport codes missing from errorTaxonomy').toEqual([])
})
+
+ it('publishes every code the proxy refuses a request with in the fixture errorTaxonomy', () => {
+ const { errorTaxonomy } = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
+ errorTaxonomy: string[]
+ }
+ const { codes, computed, sites, occurrences } = emittedRejectCodes()
+ // Liveness witness: the scanner classified every refusal a plain substring
+ // count finds, literal or allowlisted computed, and saw the direct codes.
+ expect(occurrences).toBeGreaterThanOrEqual(30)
+ expect(sites).toBe(occurrences)
+ expect([...computed].sort()).toEqual([...COMPUTED_REJECT_CODES])
+ expect([...codes]).toEqual(
+ expect.arrayContaining(['request_timeout', 'length_required', 'unsupported_media_type'])
+ )
+ const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()
+ expect(unpublished, 'codes the proxy refuses with missing from errorTaxonomy').toEqual([])
+ })
})
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index 92cd52ce9..d20266b17 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -130,6 +130,45 @@ function emittedTransportCodes(): { codes: Set<string>; sites: number; construct
return { codes, sites, constructions }
}
+// Every direct HTTP refusal in src, `reject(res, <status>, <code>)`, with its code.
+const REJECT_SITE = /reject\(res,\s*[^,()]+,\s*([^,()]+?)\s*\)/g
+// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.
+const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/
+// The only computed code a refusal may carry, as its exact source text:
+// `mapped.code` is a transport or control-api code passed through mapError,
+// whose transport half the scanner above already gates. Any other computed
+// form cannot be checked against the taxonomy, so it fails here.
+const COMPUTED_REJECT_CODES = ['mapped.code'] as const
+
+function emittedRejectCodes(): {
+ codes: Set<string>
+ computed: Set<string>
+ sites: number
+ occurrences: number
+} {
+ const codes = new Set<string>()
+ const computed = new Set<string>()
+ let sites = 0
+ let occurrences = 0
+ for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
+ const source = readFileSync(join(srcDir, name), 'utf8')
+ occurrences += source.split('reject(res,').length - 1
+ for (const match of source.matchAll(REJECT_SITE)) {
+ const code = match[1]!
+ const literal = REJECT_CODE_LITERAL.exec(code)
+ const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
+ expect(
+ literal !== null || allowed,
+ `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
+ ).toBe(true)
+ if (literal) codes.add(literal[1]!)
+ else computed.add(code)
+ sites += 1
+ }
+ }
+ return { codes, computed, sites, occurrences }
+}
+
describe('grok-subscription contract freeze', () => {
it('imports the origin-policy constants the proxy actually enforces', () => {
expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)
@@ -254,7 +293,9 @@ describe('grok-subscription contract freeze', () => {
const codes = errorTaxonomy as unknown[]
for (const code of codes) {
expect(typeof code).toBe('string')
- expect(String(code)).toMatch(/^[a-z][a-z0-9_]+$/)
+ // Snake case, except `Unauthorized`: the wire code the proxy's platform
+ // JWT check answers 401 with, published as sent.
+ expect(String(code)).toMatch(/^(?:[a-z][a-z0-9_]+|Unauthorized)$/)
}
expect(new Set(codes).size, 'errorTaxonomy must not repeat a code').toBe(codes.length)
expect(codes).toEqual(
@@ -282,6 +323,40 @@ describe('grok-subscription contract freeze', () => {
expect(unpublished, 'emitted transport codes missing from errorTaxonomy').toEqual([])
})
+ it('publishes every code the proxy refuses a request with in the fixture errorTaxonomy', () => {
+ const errorTaxonomy = readFixture().errorTaxonomy as string[]
+ const { codes, computed, sites, occurrences } = emittedRejectCodes()
+ // Liveness witness: the scanner classified every refusal a plain substring
+ // count finds, literal or allowlisted computed, and saw the direct codes.
+ expect(occurrences).toBeGreaterThanOrEqual(27)
+ expect(sites).toBe(occurrences)
+ expect([...computed].sort()).toEqual([...COMPUTED_REJECT_CODES])
+ expect([...codes]).toEqual(
+ expect.arrayContaining(['request_timeout', 'length_required', 'unsupported_media_type'])
+ )
+ const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()
+ expect(unpublished, 'codes the proxy refuses with missing from errorTaxonomy').toEqual([])
+ })
+
+ it('lists the admission codes the proxy emits or passes through in the frozen taxonomy', () => {
+ // request_timeout, length_required and unsupported_media_type are direct
+ // refusals in src/server.ts; ticket_expired is a control-api redeem code
+ // the proxy passes through (ATTEMPT_ERROR_STATUS, mapError).
+ const serverSource = readFileSync(join(srcDir, 'server.ts'), 'utf8')
+ // Witness: the pass-through table carries the code this test requires.
+ expect(serverSource).toMatch(/^\s*ticket_expired: 403,$/m)
+ const errors = readFixture().errorTaxonomy as string[]
+ expect(Array.isArray(errors) && errors.length > 0).toBe(true)
+ expect(errors).toEqual(
+ expect.arrayContaining([
+ 'request_timeout',
+ 'length_required',
+ 'ticket_expired',
+ 'unsupported_media_type',
+ ])
+ )
+ })
+
it('mirrors the exact OAuth origins control-api dials, including revoke', () => {
// control-api owns the OAuth client; the proxy never dials auth.x.ai. Cross-
// check the frozen fixture against control-api's exported constants by
diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
index 3c24e04cb..26895863f 100644
--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
@@ -316,8 +316,16 @@ describe('codex-subscription contract freeze', () => {
expect(Array.isArray(errors) && (errors as unknown[]).length > 0).toBe(true)
for (const code of errors as unknown[]) {
expect(typeof code).toBe('string')
- expect(String(code)).toMatch(/^[a-z][a-z0-9_]+$/)
+ // Snake case, except `Unauthorized`: the wire code the proxy's platform
+ // JWT check answers 401 with, published as sent.
+ expect(String(code)).toMatch(/^(?:[a-z][a-z0-9_]+|Unauthorized)$/)
}
+ // Codes the proxy emits directly (request_timeout, length_required,
+ // unsupported_media_type) or passes through from redeem (ticket_expired).
+ expect(errors).toContain('request_timeout')
+ expect(errors).toContain('length_required')
+ expect(errors).toContain('ticket_expired')
+ expect(errors).toContain('unsupported_media_type')
expect(errors).toContain('tool_call_limit_exceeded')
expect(errors).toContain('context_length_exceeded')
expect(errors).toContain('invalid_tool_arguments')
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 81e806c40..9247d0f1c 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.94",
+ "version": "0.1.95",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.94",
+ "version": "0.1.95",
"devDependencies": {
"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract",
"@types/jsonwebtoken": "^9.0.10",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 78e368b58..2f2e2cbbc 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.94",
+ "version": "0.1.95",
"private": true,
"type": "module",
"scripts": {