Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(contract-freeze): close the review gaps in the #738 LIMITS pins

Alfredo Lopez476c1cecfba5
diff --git a/Makefile b/Makefile
index d1a48e173..9e630baa7 100644
--- a/Makefile
+++ b/Makefile
@@ -1437,11 +1437,8 @@ test-e2e-stateless-idle-calibration: ## Run stateless T_idle calibration sweep (
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-idle-calibration.sh
.PHONY: test-e2e-deps
-test-e2e-deps: ## Install tests/e2e dependencies when missing
- @if [ ! -x tests/e2e/node_modules/.bin/vitest ]; then \
- echo "Installing tests/e2e dependencies with npm ci..."; \
- cd tests/e2e && npm ci --no-audit --no-fund; \
- fi
+test-e2e-deps: ## Install tests/e2e dependencies when missing or stale against the lockfile
+ @bash scripts/e2e/ensure-e2e-deps.sh tests/e2e
.PHONY: test-e2e-vitest
test-e2e-vitest: test-e2e-deps ## Run vitest-based E2E suites (tests/e2e/)
diff --git a/codex-llm-proxy/test/streamLimitsFreeze.test.ts b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
new file mode 100644
index 000000000..60762e52b
--- /dev/null
+++ b/codex-llm-proxy/test/streamLimitsFreeze.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest'
+import { readFileSync } from 'node:fs'
+import { STREAM_LIMITS } from '../src/requestLimits.js'
+
+const fixturePath = new URL(
+ '../../tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json',
+ import.meta.url
+)
+
+describe('codex-subscription stream limits freeze', () => {
+ it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
+ // StreamGate and the stream deadline enforce these three published bounds
+ // from the proxy's own constant, not from the contract package, so the
+ // Codex freeze gate in tests/e2e cannot see them.
+ const { limits } = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
+ limits: Record<string, unknown>
+ }
+ const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
+ expect(Object.keys(streamLimits).sort()).toEqual([
+ 'maxConcurrentStreams',
+ 'maxQueuedRequests',
+ 'maxStreamDurationMs',
+ ])
+ for (const name of Object.keys(streamLimits)) {
+ expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })
+ }
+ })
+})
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 4bc6e77be..4c4ab7a30 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -63,15 +63,16 @@ one physical execution per ticket. A retry or fallback must mint a new attempt.
Tool definitions have no independent count ceiling in the Evenfire request
contract. The entire serialized request, including all definitions, remains
-bounded by `maxRequestBodyBytes` (1 MiB). Every definition still undergoes
-name, schema, finite-value and unknown-field validation. `maxToolCalls` (256)
-bounds calls in each assistant history message and each newly returned
+bounded by `maxRequestBodyBytes`. Every definition still undergoes
+name, schema, finite-value and unknown-field validation. The limit values live
+only in the table above, which the freeze gate checks against the fixture and
+the runtime. `maxToolCalls` bounds calls in each assistant history message and each newly returned
response. The proxy buffers tool calls until successful completion and
validates the bound before publishing any executable call; the Host validates
it again before returning the batch. A response over the bound fails with
`tool_call_limit_exceeded`, which is not retried and does not fail over. It is
not a catalog size limit and does not widen execution concurrency.
-`maxMessages` (1024) bounds the request history. The Host rejects a longer
+`maxMessages` bounds the request history. The Host rejects a longer
history with `request_limit_exceeded` before authorization, so no ticket is
minted for it.
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index 4c2def95b..bfafd9930 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -17,6 +17,7 @@ import {
GROK_COMPLETIONS_ORIGIN,
GROK_TRANSPORT_PROTOCOL,
} from '../src/originPolicy.js'
+import { STREAM_LIMITS } from '../src/requestLimits.js'
const here = dirname(fileURLToPath(import.meta.url))
const fixturePath = join(
@@ -37,27 +38,44 @@ const FIXTURE_LIMIT_KEYS = [
'maxRetriesPerAttempt',
] as const
+// Contract `LIMITS` keys that the fixture also publishes. Each must carry the
+// same value on both sides.
+const SHARED_LIMIT_KEYS = [
+ 'maxDeadlineMs',
+ 'maxMessages',
+ 'maxOutputTokens',
+ 'maxRequestBodyBytes',
+ 'maxToolCalls',
+] as const
+
// Runtime bounds the published contract deliberately does not describe. Every
-// key of `LIMITS` must be either published (in the fixture with the same
-// value) or listed here, so adding a key to `LIMITS` fails this suite until
-// someone decides which: publish it in the fixture, or name it here.
+// key of `LIMITS` must be in exactly one of these two lists, so adding a key to
+// `LIMITS` fails this suite until someone decides which. Publishing it takes
+// three edits: the fixture's `limits`, FIXTURE_LIMIT_KEYS and
+// SHARED_LIMIT_KEYS. Keeping it runtime-only takes one: add it here.
const RUNTIME_ONLY_LIMIT_KEYS = ['maxIdLength', 'maxNestingDepth'] as const
+type Fixture = {
+ protocolVersion: string
+ origins: Record<string, string>
+ supportedOperations: string[]
+ oauthScopes: string[]
+ limits: Record<string, unknown>
+ identityHeaders: Record<string, string | boolean>
+ forbiddenOrigins: string[]
+}
+
+function readFixture(): Fixture {
+ return JSON.parse(readFileSync(fixturePath, 'utf8')) as Fixture
+}
+
describe('grok-subscription contract freeze', () => {
it('imports the origin-policy constants the proxy actually enforces', () => {
expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)
expect(GROK_CATALOG_ORIGIN).toBe(CATALOG_ORIGIN)
expect(GROK_TRANSPORT_PROTOCOL).toBe(TRANSPORT_PROTOCOL_VERSION)
- const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
- protocolVersion: string
- origins: Record<string, string>
- supportedOperations: string[]
- oauthScopes: string[]
- limits: Record<string, number>
- identityHeaders: Record<string, string | boolean>
- forbiddenOrigins: string[]
- }
+ const fixture = readFixture()
expect(fixture.protocolVersion).toBe(GROK_TRANSPORT_PROTOCOL)
expect(fixture.origins.completions).toBe(GROK_COMPLETIONS_ORIGIN)
@@ -67,45 +85,6 @@ describe('grok-subscription contract freeze', () => {
expect(fixture.origins.oauthRevoke).toBe('https://auth.x.ai/oauth2/revoke')
expect(fixture.supportedOperations).not.toContain('oauth_browser')
expect(fixture.supportedOperations).toContain('oauth_device')
- expect(fixture.limits.maxToolCalls).toBe(LIMITS.maxToolCalls)
- expect(fixture.limits.maxToolCalls).toBe(256)
- expect(fixture.limits.maxMessages).toBe(LIMITS.maxMessages)
- expect(fixture.limits.maxMessages).toBe(1024)
- expect(fixture.limits.maxRetriesPerAttempt).toBe(1)
- // Every other bound the fixture shares with the contract, cross-checked the
- // same way. Without this, a raise in `LIMITS` that forgets the fixture is
- // caught for these two fields and silently accepted for the rest.
- const shared = Object.keys(LIMITS).filter(name => name in fixture.limits)
- expect(shared).toEqual([
- 'maxRequestBodyBytes',
- 'maxMessages',
- 'maxToolCalls',
- 'maxOutputTokens',
- 'maxDeadlineMs',
- ])
- for (const name of shared) {
- expect({ [name]: fixture.limits[name] }).toEqual({
- [name]: LIMITS[name as keyof typeof LIMITS],
- })
- }
- // `shared` only sees keys the fixture already has, so a bound added to
- // `LIMITS` alone slips past it. Close that side: the fixture key set is
- // exact, and every `LIMITS` key is either shared or declared runtime-only.
- expect(Object.keys(fixture.limits).sort()).toEqual([...FIXTURE_LIMIT_KEYS].sort())
- for (const name of RUNTIME_ONLY_LIMIT_KEYS) {
- expect(
- LIMITS,
- `${name} is listed as runtime-only but LIMITS no longer has it; remove it from RUNTIME_ONLY_LIMIT_KEYS`
- ).toHaveProperty(name)
- expect(
- fixture.limits,
- `${name} is published in the fixture; remove it from RUNTIME_ONLY_LIMIT_KEYS`
- ).not.toHaveProperty(name)
- }
- expect(
- Object.keys(LIMITS).sort(),
- 'every LIMITS key must be published in the fixture or listed in RUNTIME_ONLY_LIMIT_KEYS'
- ).toEqual([...shared, ...RUNTIME_ONLY_LIMIT_KEYS].sort())
// Live xAI gates subscription inference on a client version (426 probe,
// 2026-09-18), so we send one — but the identity stays Evenfire's and never
// claims to be the Grok CLI itself.
@@ -125,6 +104,67 @@ describe('grok-subscription contract freeze', () => {
expect(GROK_CATALOG_ORIGIN).not.toContain('api.x.ai')
})
+ it('publishes an exact set of finite positive integer limits', () => {
+ const { limits } = readFixture()
+ // An unknown key is a typo or an unreviewed addition, not an extension.
+ expect(Object.keys(limits).sort()).toEqual([...FIXTURE_LIMIT_KEYS].sort())
+ for (const name of FIXTURE_LIMIT_KEYS) {
+ const value = limits[name]
+ expect(
+ Number.isSafeInteger(value) && (value as number) > 0,
+ `limits.${name} must be a finite positive integer, got ${JSON.stringify(value)}`
+ ).toBe(true)
+ }
+ expect(limits.maxToolCalls).toBe(256)
+ expect(limits.maxMessages).toBe(1024)
+ expect(limits.maxRetriesPerAttempt).toBe(1)
+ })
+
+ it('pins the contract LIMITS to the limits the fixture publishes', () => {
+ const { limits } = readFixture()
+ const runtimeLimits: Record<string, number> = { ...LIMITS }
+ const shared = Object.keys(runtimeLimits).filter(name => name in limits)
+ expect(
+ [...shared].sort(),
+ 'the LIMITS keys the fixture publishes changed; update SHARED_LIMIT_KEYS and FIXTURE_LIMIT_KEYS together'
+ ).toEqual([...SHARED_LIMIT_KEYS])
+ for (const name of shared) {
+ expect({ [name]: limits[name] }).toEqual({ [name]: runtimeLimits[name] })
+ }
+ // `shared` only sees keys the fixture already has, so a bound added to
+ // `LIMITS` alone slips past it. Close that side: every `LIMITS` key is
+ // either shared or declared runtime-only.
+ for (const name of RUNTIME_ONLY_LIMIT_KEYS) {
+ expect(
+ runtimeLimits,
+ `${name} is listed as runtime-only but LIMITS no longer has it; remove it from RUNTIME_ONLY_LIMIT_KEYS`
+ ).toHaveProperty(name)
+ expect(
+ limits,
+ `${name} is published in the fixture; remove it from RUNTIME_ONLY_LIMIT_KEYS`
+ ).not.toHaveProperty(name)
+ }
+ expect(
+ Object.keys(runtimeLimits).sort(),
+ 'every LIMITS key must be published in the fixture or listed in RUNTIME_ONLY_LIMIT_KEYS'
+ ).toEqual([...shared, ...RUNTIME_ONLY_LIMIT_KEYS].sort())
+ })
+
+ it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
+ // StreamGate and the stream deadline enforce these three published bounds
+ // from the proxy's own constant, not from the contract package.
+ const { limits } = readFixture()
+ const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
+ expect(Object.keys(streamLimits).sort()).toEqual([
+ 'maxConcurrentStreams',
+ 'maxQueuedRequests',
+ 'maxStreamDurationMs',
+ ])
+ for (const name of Object.keys(streamLimits)) {
+ expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })
+ }
+ })
+
it('mirrors the exact OAuth origins control-api dials, including revoke', () => {
// control-api owns the OAuth client; the proxy never dials auth.x.ai. Cross-
// check the frozen fixture against control-api's exported constants by
diff --git a/packages/grok-provider-attempt-contract/index.test.cjs b/packages/grok-provider-attempt-contract/index.test.cjs
index 859ad4141..a04761a2d 100644
--- a/packages/grok-provider-attempt-contract/index.test.cjs
+++ b/packages/grok-provider-attempt-contract/index.test.cjs
@@ -32,11 +32,25 @@ test('declared LIMITS literals match the runtime values', () => {
const declarations = fs.readFileSync(path.join(__dirname, 'index.d.ts'), 'utf8')
const block = declarations.match(/export declare const LIMITS: \{([\s\S]*?)\n\}/)
assert.ok(block, 'index.d.ts must declare a LIMITS object literal')
+ // Comments are stripped first, so a commented-out member cannot stand in for
+ // a real one. Every remaining line must be a member with a plain integer
+ // literal: a widened `number`, a decimal or a numeric separator fails here
+ // rather than being skipped by the scrape.
+ const members = block[1]
+ .replace(/\/\*[\s\S]*?\*\//g, '')
+ .replace(/\/\/.*$/gm, '')
+ .split('\n')
+ .map(line => line.trim())
+ .filter(line => line !== '')
const declared = Object.fromEntries(
- Array.from(block[1].matchAll(/readonly\s+([A-Za-z0-9_]+):\s*(\d+)\b/g), m => [m[1], Number(m[2])])
+ members.map(line => {
+ const m = /^readonly ([A-Za-z0-9_]+): (\d+)$/.exec(line)
+ assert.ok(m, `LIMITS member must be "readonly <name>: <integer literal>": ${line}`)
+ return [m[1], Number(m[2])]
+ })
)
- // An empty or partial scrape would otherwise pass by comparing nothing.
- assert.equal(Object.keys(declared).length, Object.keys(contract.LIMITS).length)
+ // fromEntries keeps the last of two members with the same name.
+ assert.equal(Object.keys(declared).length, members.length, 'LIMITS declares a member twice')
assert.deepEqual(declared, { ...contract.LIMITS })
})
@@ -277,6 +291,22 @@ test('LIMITS publishes the nesting depth cap', () => {
assert.equal(contract.LIMITS.maxNestingDepth, 64)
})
+test('LIMITS publishes the id length cap', () => {
+ assert.equal(contract.LIMITS.maxIdLength, 128)
+})
+
+// ID_PATTERN spells the id length out instead of reading LIMITS.maxIdLength,
+// so this ties the two together: changing either one alone fails here.
+test('request ids accept LIMITS.maxIdLength characters and reject one more', () => {
+ const max = contract.LIMITS.maxIdLength
+ assert.equal(contract.parseGrokCompletionRequestV1({ ...BASE, requestId: 'r'.repeat(max) }).ok, true)
+ assert.deepEqual(contract.parseGrokCompletionRequestV1({ ...BASE, requestId: 'r'.repeat(max + 1) }), {
+ ok: false,
+ code: 'invalid',
+ message: 'requestId is invalid',
+ })
+})
+
test('tool parameters accept depth 64 and reject depth 65 with a limit failure', () => {
const at = contract.parseGrokCompletionRequestV1({
...BASE,
diff --git a/packages/llm-provider-attempt-contract/index.test.cjs b/packages/llm-provider-attempt-contract/index.test.cjs
index ee8482825..6c7f8e155 100644
--- a/packages/llm-provider-attempt-contract/index.test.cjs
+++ b/packages/llm-provider-attempt-contract/index.test.cjs
@@ -37,11 +37,25 @@ test('declared LIMITS literals match the runtime values', () => {
const declarations = fs.readFileSync(path.join(__dirname, 'index.d.ts'), 'utf8')
const block = declarations.match(/export declare const LIMITS: \{([\s\S]*?)\n\}/)
assert.ok(block, 'index.d.ts must declare a LIMITS object literal')
+ // Comments are stripped first, so a commented-out member cannot stand in for
+ // a real one. Every remaining line must be a member with a plain integer
+ // literal: a widened `number`, a decimal or a numeric separator fails here
+ // rather than being skipped by the scrape.
+ const members = block[1]
+ .replace(/\/\*[\s\S]*?\*\//g, '')
+ .replace(/\/\/.*$/gm, '')
+ .split('\n')
+ .map(line => line.trim())
+ .filter(line => line !== '')
const declared = Object.fromEntries(
- Array.from(block[1].matchAll(/readonly\s+([A-Za-z0-9_]+):\s*(\d+)\b/g), m => [m[1], Number(m[2])])
+ members.map(line => {
+ const m = /^readonly ([A-Za-z0-9_]+): (\d+)$/.exec(line)
+ assert.ok(m, `LIMITS member must be "readonly <name>: <integer literal>": ${line}`)
+ return [m[1], Number(m[2])]
+ })
)
- // An empty or partial scrape would otherwise pass by comparing nothing.
- assert.equal(Object.keys(declared).length, Object.keys(contract.LIMITS).length)
+ // fromEntries keeps the last of two members with the same name.
+ assert.equal(Object.keys(declared).length, members.length, 'LIMITS declares a member twice')
assert.deepEqual(declared, { ...contract.LIMITS })
})
@@ -546,6 +560,22 @@ test('LIMITS publishes the nesting depth cap', () => {
assert.equal(contract.LIMITS.maxNestingDepth, 64)
})
+test('LIMITS publishes the id length cap', () => {
+ assert.equal(contract.LIMITS.maxIdLength, 128)
+})
+
+// ID_PATTERN spells the id length out instead of reading LIMITS.maxIdLength,
+// so this ties the two together: changing either one alone fails here.
+test('request ids accept LIMITS.maxIdLength characters and reject one more', () => {
+ const max = contract.LIMITS.maxIdLength
+ assert.equal(contract.parseCodexCompletionRequestV1({ ...BASE, requestId: 'r'.repeat(max) }).ok, true)
+ assert.deepEqual(contract.parseCodexCompletionRequestV1({ ...BASE, requestId: 'r'.repeat(max + 1) }), {
+ ok: false,
+ code: 'invalid',
+ message: 'requestId is invalid',
+ })
+})
+
test('tool parameters accept depth 64 and reject depth 65 with a limit failure', () => {
const at = contract.parseCodexCompletionRequestV1({
...BASE,
diff --git a/scripts/e2e/ensure-e2e-deps.sh b/scripts/e2e/ensure-e2e-deps.sh
new file mode 100644
index 000000000..4758fa0e1
--- /dev/null
+++ b/scripts/e2e/ensure-e2e-deps.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Install tests/e2e dependencies with `npm ci` when they are missing or were
+# installed from a different lockfile.
+#
+# Checking only for node_modules/.bin/vitest keeps a node_modules installed
+# before a lockfile change, so a newly added dependency (a `file:` link to a
+# contract package, say) fails later as a missing import. After a successful
+# `npm ci` this copies the lockfile into node_modules; any difference from the
+# current lockfile triggers a fresh `npm ci`, which replaces node_modules.
+#
+# Usage: ensure-e2e-deps.sh <e2e-dir>
+
+set -euo pipefail
+
+E2E_DIR="${1:?usage: ensure-e2e-deps.sh <e2e-dir>}"
+LOCKFILE="${E2E_DIR}/package-lock.json"
+INSTALLED_LOCKFILE="${E2E_DIR}/node_modules/.installed-package-lock.json"
+
+[[ -f "${LOCKFILE}" ]] || { echo "[e2e-deps] ERROR: missing ${LOCKFILE}" >&2; exit 1; }
+
+if [[ -x "${E2E_DIR}/node_modules/.bin/vitest" ]] && cmp -s "${LOCKFILE}" "${INSTALLED_LOCKFILE}"; then
+ exit 0
+fi
+
+echo "[e2e-deps] Installing ${E2E_DIR} dependencies with npm ci"
+(cd "${E2E_DIR}" && npm ci --no-audit --no-fund)
+cp "${LOCKFILE}" "${INSTALLED_LOCKFILE}"
diff --git a/scripts/e2e/run-vitest-e2e.sh b/scripts/e2e/run-vitest-e2e.sh
index 7b489c00b..22830419f 100755
--- a/scripts/e2e/run-vitest-e2e.sh
+++ b/scripts/e2e/run-vitest-e2e.sh
@@ -83,10 +83,7 @@ die() {
}
ensure_vitest_dependencies() {
- if [[ ! -x tests/e2e/node_modules/.bin/vitest ]]; then
- log "Installing tests/e2e dependencies with npm ci"
- (cd tests/e2e && npm ci --no-audit --no-fund)
- fi
+ bash "${SCRIPT_DIR}/ensure-e2e-deps.sh" tests/e2e
}
# Run Vitest and FAIL if zero tests actually executed, even on a green exit:
diff --git a/scripts/tests/test-e2e-vitest-node-unit-gate.sh b/scripts/tests/test-e2e-vitest-node-unit-gate.sh
index 36fc87b4b..eb4cf9c99 100755
--- a/scripts/tests/test-e2e-vitest-node-unit-gate.sh
+++ b/scripts/tests/test-e2e-vitest-node-unit-gate.sh
@@ -114,5 +114,56 @@ require_step_gate "${WORKFLOW}" "Test image-capabilities fixtures and runner" \
"image-capabilities step mcp-host gate"
require_step_after "${WORKFLOW}" "Test image-capabilities fixtures and runner" "Build" \
"image-capabilities step ordered after Build in the same job"
+require_contains "${RUNNER}" 'bash "${SCRIPT_DIR}/ensure-e2e-deps.sh" tests/e2e' \
+ "runner dependency install through the lockfile-aware helper"
+require_contains "${ROOT_DIR}/Makefile" "bash scripts/e2e/ensure-e2e-deps.sh tests/e2e" \
+ "test-e2e-deps install through the lockfile-aware helper"
+
+# Behaviour of the install helper against a fake npm that records each call and
+# lays down node_modules/.bin/vitest the way `npm ci` would.
+check_ensure_e2e_deps() {
+ local tmp
+ tmp="$(mktemp -d)"
+ trap 'rm -rf "${tmp}"' RETURN
+ mkdir -p "${tmp}/bin" "${tmp}/e2e"
+ cat > "${tmp}/bin/npm" <<'FAKE_NPM'
+#!/usr/bin/env bash
+echo "npm $*" >> "${FAKE_NPM_LOG}"
+[[ "${FAKE_NPM_FAIL:-0}" == "1" ]] && exit 7
+rm -rf node_modules
+mkdir -p node_modules/.bin
+printf '#!/bin/sh\n' > node_modules/.bin/vitest
+chmod +x node_modules/.bin/vitest
+FAKE_NPM
+ chmod +x "${tmp}/bin/npm"
+ echo '{"lockfileVersion":3,"v":1}' > "${tmp}/e2e/package-lock.json"
+
+ local helper="${ROOT_DIR}/scripts/e2e/ensure-e2e-deps.sh"
+ local log="${tmp}/npm.log"
+ run_helper() { PATH="${tmp}/bin:${PATH}" FAKE_NPM_LOG="${log}" bash "${helper}" "${tmp}/e2e" > /dev/null; }
+ npm_calls() { if [[ -f "${log}" ]]; then wc -l < "${log}" | tr -d ' '; else echo 0; fi; }
+
+ run_helper
+ [[ "$(npm_calls)" == "1" ]] || { echo "ensure-e2e-deps: fresh tree did not run npm ci" >&2; exit 1; }
+ grep -Fxq "npm ci --no-audit --no-fund" "${log}" || { echo "ensure-e2e-deps: unexpected npm invocation" >&2; exit 1; }
+
+ run_helper
+ [[ "$(npm_calls)" == "1" ]] || { echo "ensure-e2e-deps: unchanged lockfile reinstalled" >&2; exit 1; }
+
+ echo '{"lockfileVersion":3,"v":2}' > "${tmp}/e2e/package-lock.json"
+ run_helper
+ [[ "$(npm_calls)" == "2" ]] || { echo "ensure-e2e-deps: changed lockfile did not reinstall" >&2; exit 1; }
+
+ # A failed install must fail the helper and leave no record of success.
+ echo '{"lockfileVersion":3,"v":3}' > "${tmp}/e2e/package-lock.json"
+ if PATH="${tmp}/bin:${PATH}" FAKE_NPM_LOG="${log}" FAKE_NPM_FAIL=1 bash "${helper}" "${tmp}/e2e" > /dev/null 2>&1; then
+ echo "ensure-e2e-deps: a failed npm ci was reported as success" >&2
+ exit 1
+ fi
+ [[ "$(npm_calls)" == "3" ]] || { echo "ensure-e2e-deps: failing install was not attempted" >&2; exit 1; }
+ run_helper
+ [[ "$(npm_calls)" == "4" ]] || { echo "ensure-e2e-deps: install after a failed one was skipped" >&2; exit 1; }
+}
+check_ensure_e2e_deps
echo "E2E Vitest node-unit public gate contract OK"
diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
index 6d2f0a17a..aa4e4844b 100644
--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
@@ -48,11 +48,22 @@ const REQUIRED_LIMIT_KEYS = [
'maxRetriesPerAttempt',
] as const
+// Runtime `LIMITS` keys that the fixture also publishes. Each must carry the
+// same value on both sides.
+const PUBLISHED_RUNTIME_LIMIT_KEYS = [
+ 'maxDeadlineMs',
+ 'maxMessages',
+ 'maxOutputTokens',
+ 'maxRequestBodyBytes',
+ 'maxToolCalls',
+] as const
+
// Runtime bounds the published contract deliberately does not describe. Every
-// key of the runtime `LIMITS` must be either published (present in the fixture
-// with the same value) or listed here, so adding a key to `LIMITS` fails this
-// suite until someone decides which: publish it in the fixture and the
-// architecture doc, or name it here as runtime-only.
+// key of the runtime `LIMITS` must be in exactly one of these two lists, so
+// adding a key to `LIMITS` fails this suite until someone decides which.
+// Publishing it takes four edits: the fixture's `limits`, REQUIRED_LIMIT_KEYS,
+// PUBLISHED_RUNTIME_LIMIT_KEYS and the architecture doc table. Keeping it
+// runtime-only takes one: add it here.
const RUNTIME_ONLY_LIMIT_KEYS = ['maxIdLength', 'maxNestingDepth'] as const
const SENSITIVE_VALUE_PATTERN =
@@ -75,6 +86,30 @@ function assertFinitePositiveLimit(value: unknown, key: string): asserts value i
expect(n, `${key} must be > 0 (missing/0/negative/unlimited are invalid)`).toBeGreaterThan(0)
}
+/**
+ * Rows of the architecture doc's `| Limit | Value |` table, in document order.
+ * The table ends at the first line that is not a table row.
+ */
+function parseLimitsTable(doc: string): Array<[string, string]> {
+ const lines = doc.split('\n')
+ const header = lines.findIndex(line => /^\|\s*Limit\s*\|\s*Value\s*\|$/.test(line))
+ expect(
+ header,
+ 'architecture doc must contain a "| Limit | Value |" table'
+ ).toBeGreaterThanOrEqual(0)
+ expect(lines[header + 1], 'limits table header must be followed by a separator row').toMatch(
+ /^\|\s*-+\s*\|\s*-+\s*\|$/
+ )
+ const rows: Array<[string, string]> = []
+ for (const line of lines.slice(header + 2)) {
+ if (!line.startsWith('|')) break
+ const cells = /^\|\s*([A-Za-z0-9]+)\s*\|\s*(\S+)\s*\|$/.exec(line)
+ expect(cells, `malformed limits table row: ${line}`).not.toBeNull()
+ rows.push([cells![1], cells![2]])
+ }
+ return rows
+}
+
function collectSensitiveLeaves(value: unknown, path: string, hits: string[]): void {
if (value === null || value === undefined) return
if (typeof value === 'string') {
@@ -170,13 +205,10 @@ describe('codex-subscription contract freeze', () => {
// in LIMITS and left behind here passes every gate (#738).
const runtimeLimits: Record<string, number> = { ...LIMITS }
const published = Object.keys(runtimeLimits).filter(key => key in (limits ?? {}))
- expect(published).toEqual([
- 'maxRequestBodyBytes',
- 'maxMessages',
- 'maxToolCalls',
- 'maxOutputTokens',
- 'maxDeadlineMs',
- ])
+ expect(
+ [...published].sort(),
+ 'the LIMITS keys the fixture publishes changed; update PUBLISHED_RUNTIME_LIMIT_KEYS, REQUIRED_LIMIT_KEYS and the architecture doc table together'
+ ).toEqual([...PUBLISHED_RUNTIME_LIMIT_KEYS])
for (const key of RUNTIME_ONLY_LIMIT_KEYS) {
expect(
runtimeLimits,
@@ -200,12 +232,19 @@ describe('codex-subscription contract freeze', () => {
expect(limits?.maxRequestBodyBytes).toBe(1048576)
// The architecture doc publishes the same limits as a table; a row that
- // drifts from the fixture misdescribes what the runtime enforces.
- for (const key of REQUIRED_LIMIT_KEYS) {
- const value = String(limits?.[key])
- expect(architectureDoc, `architecture doc limits table must list ${key} = ${value}`).toMatch(
- new RegExp(`^\\| ${key} +\\| ${value} +\\|$`, 'm')
- )
+ // drifts from the fixture misdescribes what the runtime enforces. The table
+ // is parsed as a whole so a duplicated, extra or missing row also fails.
+ const docLimits = parseLimitsTable(architectureDoc)
+ const docKeys = docLimits.map(([key]) => key)
+ expect(
+ [...docKeys].sort(),
+ 'architecture doc limits table must list each fixture limit exactly once'
+ ).toEqual([...REQUIRED_LIMIT_KEYS].sort())
+ for (const [key, value] of docLimits) {
+ expect(
+ { [key]: value },
+ `architecture doc limits table must list ${key} = ${String(limits?.[key])}`
+ ).toEqual({ [key]: String(limits?.[key]) })
}
const errors = contract.errorTaxonomy
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index c342dfc46..26f0686c2 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.87",
+ "version": "0.1.88",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.87",
+ "version": "0.1.88",
"devDependencies": {
"@clerum/llm-provider-attempt-contract": "file:../../packages/llm-provider-attempt-contract",
"@types/jsonwebtoken": "^9.0.10",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 99e30ff30..6a3592a52 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.87",
+ "version": "0.1.88",
"private": true,
"type": "module",
"scripts": {