Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
21.6 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { readFileSync, readdirSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import ts from 'typescript'
import {
  CATALOG_ORIGIN,
  COMPLETIONS_ORIGIN,
  LIMITS,
  TRANSPORT_PROTOCOL_VERSION,
} from '@clerum/grok-provider-attempt-contract'
import {
  GROK_UPSTREAM_CLIENT_IDENTIFIER,
  GROK_UPSTREAM_USER_AGENT,
} from '../src/grokUpstreamHeaders.js'
import {
  GROK_CATALOG_ORIGIN,
  GROK_COMPLETIONS_ORIGIN,
  GROK_TRANSPORT_PROTOCOL,
} from '../src/originPolicy.js'
import { STREAM_LIMITS } from '../src/requestLimits.js'

const here = dirname(fileURLToPath(import.meta.url))
const fixturePath = join(
  here,
  '../../tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json'
)
const architectureDocPath = join(
  here,
  '../../docs/architecture/grok-subscription-transport-contract.md'
)

// The limits the frozen fixture publishes, exactly.
const FIXTURE_LIMIT_KEYS = [
  'maxRequestBodyBytes',
  'maxMessages',
  'maxToolCalls',
  'maxOutputTokens',
  'maxStreamDurationMs',
  'maxDeadlineMs',
  'maxConcurrentStreams',
  'maxQueuedRequests',
  'maxQueueWaitMs',
  'upstreamIdleTimeoutMs',
  'maxRetriesPerAttempt',
  'executionTicketTtlMs',
] as const

// Contract `LIMITS` keys that the fixture also publishes. Each must carry the
// same value on both sides.
const SHARED_LIMIT_KEYS = [
  'executionTicketTtlMs',
  'maxDeadlineMs',
  'maxMessages',
  'maxOutputTokens',
  'maxRequestBodyBytes',
  'maxToolCalls',
] as const

// Runtime bounds the published contract deliberately does not describe. Every
// key of `LIMITS` must be in exactly one of these two lists, so adding a key to
// `LIMITS` fails this suite until someone decides which. Publishing it takes
// three edits: the fixture's `limits`, FIXTURE_LIMIT_KEYS and
// SHARED_LIMIT_KEYS. Keeping it runtime-only takes one: add it here.
// maxVisualRequestBodyBytes stays here while the subscription endpoint is
// unmeasured for images: the fixture describes the measured upstream.
// maxRequestContainers bounds the proxies' parse memory (JSON.parse allocates
// one heap object per container), not the upstream.
const RUNTIME_ONLY_LIMIT_KEYS = [
  'maxIdLength',
  'maxNestingDepth',
  'maxVisualRequestBodyBytes',
  'maxRequestContainers',
  'maxRequestMembers',
  'maxRequestElements',
] as const

type Fixture = {
  protocolVersion: string
  origins: Record<string, string>
  supportedOperations: string[]
  oauthScopes: string[]
  limits: Record<string, unknown>
  identityHeaders: Record<string, string | boolean>
  forbiddenOrigins: string[]
  errorTaxonomy: unknown
}

function readFixture(): Fixture {
  return JSON.parse(readFileSync(fixturePath, 'utf8')) as Fixture
}

/**
 * Rows of the architecture doc's `| Limit | Value |` table, in document order.
 * The table ends at the first line that is not a table row.
 */
function parseLimitsTable(doc: string): Array<[string, string]> {
  const lines = doc.split('\n')
  const header = lines.findIndex(line => /^\|\s*Limit\s*\|\s*Value\s*\|$/.test(line))
  expect(
    header,
    'architecture doc must contain a "| Limit | Value |" table'
  ).toBeGreaterThanOrEqual(0)
  expect(lines[header + 1], 'limits table header must be followed by a separator row').toMatch(
    /^\|\s*-+\s*\|\s*-+\s*\|$/
  )
  const rows: Array<[string, string]> = []
  for (const line of lines.slice(header + 2)) {
    if (!line.startsWith('|')) break
    const cells = /^\|\s*([A-Za-z0-9]+)\s*\|\s*(\S+)\s*\|$/.exec(line)
    expect(cells, `malformed limits table row: ${line}`).not.toBeNull()
    rows.push([cells![1]!, cells![2]!])
  }
  return rows
}

const srcDir = join(here, '../src')

// Every construction of a transport error in src, with its first code argument.
// UpstreamTimeoutError takes its metric kind first and its wire code second.
const TRANSPORT_ERROR_SITE =
  /new (?:GrokTransportError|UpstreamTimeoutError)\(\s*(?:'(?:idle|total)',\s*)?([^,)\s]+)/g

function emittedTransportCodes(): { codes: Set<string>; sites: number; constructions: number } {
  const codes = new Set<string>()
  let sites = 0
  let constructions = 0
  for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
    const source = readFileSync(join(srcDir, name), 'utf8')
    constructions +=
      source.split('new GrokTransportError(').length -
      1 +
      source.split('new UpstreamTimeoutError(').length -
      1
    for (const match of source.matchAll(TRANSPORT_ERROR_SITE)) {
      const literal = /^'([a-z][a-z0-9_]+)'$/.exec(match[1]!)
      // A computed code cannot be checked against the taxonomy, so it fails here.
      expect(
        literal,
        `${name}: transport error code must be a string literal, got ${match[1]}`
      ).not.toBeNull()
      codes.add(literal![1]!)
      sites += 1
    }
  }
  return { codes, sites, constructions }
}

// Every direct HTTP refusal in src, `reject(res, <status>, <code>)`, with its code.
// The body-parser error handler also passes `type`, the parser's fixed error
// type, which is logged and never sent, so it is the one fourth argument allowed.
const REJECT_SITE = /reject\(res,\s*[^,()]+,\s*([^,()]+?)\s*(?:,\s*type\s*)?\)/g
// Refusal codes are wire strings; `Unauthorized` is the one spelled in capitals.
const REJECT_CODE_LITERAL = /^'([A-Za-z][A-Za-z0-9_]+)'$/
// The computed refusal forms are closed RequestLimitError codes and mapped errors.
// `mapped.code` is a transport or control-api code passed through mapError,
// whose transport half the scanner above already gates. Any other computed
// form cannot be checked against the taxonomy, so it fails here.
const COMPUTED_REJECT_CODES = ['err.code', 'mapped.code'] as const

/** Computed admission codes are allowed only through this closed, readonly type. */
function requestLimitCodes(
  sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')
): Set<string> {
  const source = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
  const alias = source.statements.find(
    (node): node is ts.TypeAliasDeclaration =>
      ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
  )
  if (!alias || !ts.isUnionTypeNode(alias.type)) {
    throw new Error('RequestLimitCode must remain a closed literal union')
  }
  const codes = new Set(
    alias.type.types.map(node => {
      if (!ts.isLiteralTypeNode(node) || !ts.isStringLiteral(node.literal)) {
        throw new Error('RequestLimitCode must not accept computed or open string types')
      }
      return node.literal.text
    })
  )
  const owner = source.statements.find(
    (node): node is ts.ClassDeclaration =>
      ts.isClassDeclaration(node) && node.name?.text === 'RequestLimitError'
  )
  const property = owner?.members.find(
    (node): node is ts.PropertyDeclaration =>
      ts.isPropertyDeclaration(node) && node.name.getText(source) === 'code'
  )
  expect(property?.type?.getText(source)).toBe('RequestLimitCode')
  expect(property?.modifiers?.some(node => node.kind === ts.SyntaxKind.ReadonlyKeyword)).toBe(true)
  const constructor = owner?.members.find(ts.isConstructorDeclaration)
  const input = constructor?.parameters.find(node => node.name.getText(source) === 'code')
  expect(input?.type?.getText(source)).toBe('RequestLimitCode')
  expect(input?.initializer?.getText(source)).toBe("'provider_unavailable'")
  const assignments: string[] = []
  const collect = (node: ts.Node): void => {
    if (
      ts.isBinaryExpression(node) &&
      node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
      node.left.getText(source) === 'this.code'
    ) {
      assignments.push(node.right.getText(source))
    }
    ts.forEachChild(node, collect)
  }
  if (owner) collect(owner)
  expect(assignments).toEqual(['code'])
  return codes
}

function rejectCodeLiteral(code: string, name: string): RegExpExecArray | null {
  const literal = REJECT_CODE_LITERAL.exec(code)
  const allowed = (COMPUTED_REJECT_CODES as readonly string[]).includes(code)
  expect(
    literal !== null || allowed,
    `${name}: reject code must be a string literal or one of ${COMPUTED_REJECT_CODES.join(', ')}, got ${code}`
  ).toBe(true)
  return literal
}

function emittedRejectCodes(): {
  codes: Set<string>
  computed: Set<string>
  sites: number
  occurrences: number
} {
  const codes = requestLimitCodes()
  const computed = new Set<string>()
  let sites = 0
  let occurrences = 0
  for (const name of readdirSync(srcDir).filter(file => file.endsWith('.ts'))) {
    const source = readFileSync(join(srcDir, name), 'utf8')
    occurrences += source.split('reject(res,').length - 1
    for (const match of source.matchAll(REJECT_SITE)) {
      const code = match[1]!
      const literal = rejectCodeLiteral(code, name)
      if (literal) codes.add(literal[1]!)
      else computed.add(code)
      sites += 1
    }
  }
  return { codes, computed, sites, occurrences }
}

describe('grok-subscription contract freeze', () => {
  it('rejects open or mutable admission code producers while reading the closed production union', () => {
    const sourceText = readFileSync(join(srcDir, 'requestLimits.ts'), 'utf8')
    const tree = ts.createSourceFile('requestLimits.ts', sourceText, ts.ScriptTarget.Latest, true)
    const alias = tree.statements.find(
      (node): node is ts.TypeAliasDeclaration =>
        ts.isTypeAliasDeclaration(node) && node.name.text === 'RequestLimitCode'
    )!
    const withType = (type: string) =>
      sourceText.slice(0, alias.type.getStart(tree)) + type + sourceText.slice(alias.type.end)
    expect([...requestLimitCodes(sourceText)]).toEqual(
      expect.arrayContaining(['provider_unavailable', 'proxy_capacity_exceeded', 'visual_gate'])
    )
    expect(() => requestLimitCodes(withType('string'))).toThrow('closed literal union')
    expect(() => requestLimitCodes(withType("'provider_unavailable' | string"))).toThrow(
      'open string types'
    )
    expect(() =>
      requestLimitCodes(
        sourceText.replace('readonly code: RequestLimitCode', 'code: RequestLimitCode')
      )
    ).toThrow()
    expect(() =>
      requestLimitCodes(sourceText.replace('this.code = code', "this.code = 'arbitrary_code'"))
    ).toThrow()
  })

  it.each(['caller.code', "err.code || 'provider_unavailable'", 'String(err.code)'])(
    'rejects unbounded computed refusal expression %s',
    expression => {
      expect(rejectCodeLiteral("'request_timeout'", 'fixture')).not.toBeNull()
      expect(rejectCodeLiteral('err.code', 'fixture')).toBeNull()
      expect(rejectCodeLiteral('mapped.code', 'fixture')).toBeNull()
      expect(() => rejectCodeLiteral(expression, 'fixture')).toThrow('reject code must be')
    }
  )

  it('imports the origin-policy constants the proxy actually enforces', () => {
    expect(GROK_COMPLETIONS_ORIGIN).toBe(COMPLETIONS_ORIGIN)
    expect(GROK_CATALOG_ORIGIN).toBe(CATALOG_ORIGIN)
    expect(GROK_TRANSPORT_PROTOCOL).toBe(TRANSPORT_PROTOCOL_VERSION)

    const fixture = readFixture()

    expect(fixture.protocolVersion).toBe(GROK_TRANSPORT_PROTOCOL)
    expect(fixture.origins.completions).toBe(GROK_COMPLETIONS_ORIGIN)
    expect(fixture.origins.catalog).toBe(GROK_CATALOG_ORIGIN)
    expect(fixture.origins.oauthDevice).toBe('https://auth.x.ai/oauth2/device/code')
    expect(fixture.origins.oauthToken).toBe('https://auth.x.ai/oauth2/token')
    expect(fixture.origins.oauthRevoke).toBe('https://auth.x.ai/oauth2/revoke')
    expect(fixture.supportedOperations).not.toContain('oauth_browser')
    expect(fixture.supportedOperations).toContain('oauth_device')
    // Live xAI gates subscription inference on a client version (426 probe,
    // 2026-09-18), so we send one — but the identity stays Evenfire's and never
    // claims to be the Grok CLI itself.
    expect(String(fixture.identityHeaders['user-agent'])).toContain(GROK_UPSTREAM_USER_AGENT)
    expect(fixture.identityHeaders['x-grok-client-identifier']).toBe(
      GROK_UPSTREAM_CLIENT_IDENTIFIER
    )
    expect(fixture.identityHeaders['x-xai-token-auth']).toBe('xai-grok-cli')
    expect(fixture.identityHeaders.cliImpersonation).toBe(false)
    expect(JSON.stringify(fixture.identityHeaders)).not.toContain('grok-shell')
    expect(JSON.stringify(fixture.identityHeaders)).not.toContain('xai-grok-workspace')
    expect(fixture.oauthScopes).toEqual(
      expect.arrayContaining(['openid', 'offline_access', 'grok-cli:access', 'api:access'])
    )
    expect(fixture.forbiddenOrigins).toEqual(expect.arrayContaining(['https://api.x.ai']))
    expect(GROK_COMPLETIONS_ORIGIN).not.toContain('api.x.ai')
    expect(GROK_CATALOG_ORIGIN).not.toContain('api.x.ai')
  })

  it('publishes an exact set of finite positive integer limits', () => {
    const { limits } = readFixture()
    // An unknown key is a typo or an unreviewed addition, not an extension.
    expect(Object.keys(limits).sort()).toEqual([...FIXTURE_LIMIT_KEYS].sort())
    for (const name of FIXTURE_LIMIT_KEYS) {
      const value = limits[name]
      expect(
        Number.isSafeInteger(value) && (value as number) > 0,
        `limits.${name} must be a finite positive integer, got ${JSON.stringify(value)}`
      ).toBe(true)
    }
    expect(limits.maxToolCalls).toBe(256)
    expect(limits.maxMessages).toBe(1024)
    expect(limits.maxRetriesPerAttempt).toBe(1)
    // control-api derives the Grok execution ticket TTL from LIMITS (#739).
    expect(limits.executionTicketTtlMs).toBe(60000)
  })

  it('pins the architecture doc limits table to the limits the fixture publishes', () => {
    const { limits } = readFixture()
    // A row that drifts from the fixture misdescribes what the runtime
    // enforces. The table is parsed as a whole, so a duplicated, extra or
    // missing row also fails.
    const docLimits = parseLimitsTable(readFileSync(architectureDocPath, 'utf8'))
    expect(
      docLimits.map(([key]) => key).sort(),
      'architecture doc limits table must list each fixture limit exactly once'
    ).toEqual([...FIXTURE_LIMIT_KEYS].sort())
    for (const [key, value] of docLimits) {
      expect(
        { [key]: value },
        `architecture doc limits table must list ${key} = ${String(limits[key])}`
      ).toEqual({ [key]: String(limits[key]) })
    }
  })

  it('pins the contract LIMITS to the limits the fixture publishes', () => {
    const { limits } = readFixture()
    const runtimeLimits: Record<string, number> = { ...LIMITS }
    const shared = Object.keys(runtimeLimits).filter(name => name in limits)
    expect(
      [...shared].sort(),
      'the LIMITS keys the fixture publishes changed; update SHARED_LIMIT_KEYS and FIXTURE_LIMIT_KEYS together'
    ).toEqual([...SHARED_LIMIT_KEYS])
    for (const name of shared) {
      expect({ [name]: limits[name] }).toEqual({ [name]: runtimeLimits[name] })
    }
    // `shared` only sees keys the fixture already has, so a bound added to
    // `LIMITS` alone slips past it. Close that side: every `LIMITS` key is
    // either shared or declared runtime-only.
    for (const name of RUNTIME_ONLY_LIMIT_KEYS) {
      expect(
        runtimeLimits,
        `${name} is listed as runtime-only but LIMITS no longer has it; remove it from RUNTIME_ONLY_LIMIT_KEYS`
      ).toHaveProperty(name)
      expect(
        limits,
        `${name} is published in the fixture; remove it from RUNTIME_ONLY_LIMIT_KEYS`
      ).not.toHaveProperty(name)
    }
    expect(
      Object.keys(runtimeLimits).sort(),
      'every LIMITS key must be published in the fixture or listed in RUNTIME_ONLY_LIMIT_KEYS'
    ).toEqual([...shared, ...RUNTIME_ONLY_LIMIT_KEYS].sort())
  })

  it('pins the proxy STREAM_LIMITS to the limits the fixture publishes', () => {
    // StreamGate and the upstream deadlines enforce these published bounds
    // from the proxy's own constant, not from the contract package.
    const { limits } = readFixture()
    const streamLimits: Record<string, number> = { ...STREAM_LIMITS }
    expect(Object.keys(streamLimits).sort()).toEqual([
      'maxConcurrentStreams',
      'maxQueueWaitMs',
      'maxQueuedRequests',
      'maxStreamDurationMs',
      'upstreamIdleTimeoutMs',
    ])
    for (const name of Object.keys(streamLimits)) {
      expect({ [name]: limits[name] }).toEqual({ [name]: streamLimits[name] })
    }
  })

  it('publishes a well-formed errorTaxonomy with the codes the transport contract names', () => {
    const { errorTaxonomy } = readFixture()
    expect(Array.isArray(errorTaxonomy) && errorTaxonomy.length > 0).toBe(true)
    const codes = errorTaxonomy as unknown[]
    for (const code of codes) {
      expect(typeof code).toBe('string')
      // Snake case, except `Unauthorized`: the wire code the proxy's platform
      // JWT check answers 401 with, published as sent.
      expect(String(code)).toMatch(/^(?:[a-z][a-z0-9_]+|Unauthorized)$/)
    }
    expect(new Set(codes).size, 'errorTaxonomy must not repeat a code').toBe(codes.length)
    expect(codes).toEqual(
      expect.arrayContaining([
        'tool_call_limit_exceeded',
        'tool_call_arguments_exceeded',
        'client_upgrade_required',
        'sse_buffer_exceeded',
        'stream_duration_exceeded',
      ])
    )
  })

  it('publishes every transport error code the proxy emits in the fixture errorTaxonomy', () => {
    const errorTaxonomy = readFixture().errorTaxonomy as string[]
    const { codes, sites, constructions } = emittedTransportCodes()
    // Liveness witness: the pattern read the code of every construction a
    // plain substring count finds, including both upstream timeout codes.
    expect(constructions).toBeGreaterThanOrEqual(15)
    expect(sites).toBe(constructions)
    expect([...codes]).toEqual(
      expect.arrayContaining(['provider_unavailable', 'stream_duration_exceeded'])
    )
    const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()
    expect(unpublished, 'emitted transport codes missing from errorTaxonomy').toEqual([])
  })

  it('publishes every code the proxy refuses a request with in the fixture errorTaxonomy', () => {
    const errorTaxonomy = readFixture().errorTaxonomy as string[]
    const { codes, computed, sites, occurrences } = emittedRejectCodes()
    // Liveness witness: the scanner classified every refusal a plain substring
    // count finds, literal or allowlisted computed, and saw the direct codes.
    expect(occurrences).toBeGreaterThanOrEqual(27)
    expect(sites).toBe(occurrences)
    expect([...computed].sort()).toEqual([...COMPUTED_REJECT_CODES])
    expect([...codes]).toEqual(
      expect.arrayContaining(['request_timeout', 'length_required', 'unsupported_media_type'])
    )
    const unpublished = [...codes].filter(code => !errorTaxonomy.includes(code)).sort()
    expect(unpublished, 'codes the proxy refuses with missing from errorTaxonomy').toEqual([])
  })

  it('lists the admission codes the proxy emits or passes through in the frozen taxonomy', () => {
    // request_timeout, length_required and unsupported_media_type are direct
    // refusals in src/server.ts; ticket_expired is both a direct refusal for
    // an authentic expired ticket and a control-api redeem code the proxy
    // passes through (ATTEMPT_ERROR_STATUS, mapError).
    const serverSource = readFileSync(join(srcDir, 'server.ts'), 'utf8')
    // Witness: the pass-through table carries the code this test requires.
    expect(serverSource).toMatch(/^\s*ticket_expired: 403,$/m)
    const errors = readFixture().errorTaxonomy as string[]
    expect(Array.isArray(errors) && errors.length > 0).toBe(true)
    expect(errors).toEqual(
      expect.arrayContaining([
        'request_timeout',
        'length_required',
        'ticket_expired',
        'unsupported_media_type',
      ])
    )
  })

  it('mirrors the exact OAuth origins control-api dials, including revoke', () => {
    // control-api owns the OAuth client; the proxy never dials auth.x.ai. Cross-
    // check the frozen fixture against control-api's exported constants by
    // source so a drift in either side fails here without importing control-api.
    const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
      origins: Record<string, string>
    }
    const oauthSource = readFileSync(
      join(here, '../../control-api/src/services/grokSubscriptionOAuth.ts'),
      'utf8'
    )
    const constant = (name: string): string | undefined =>
      new RegExp(`export const ${name} = '([^']+)'`).exec(oauthSource)?.[1]
    expect(constant('GROK_OAUTH_DEVICE_URL')).toBe(fixture.origins.oauthDevice)
    expect(constant('GROK_OAUTH_TOKEN_URL')).toBe(fixture.origins.oauthToken)
    expect(constant('GROK_OAUTH_REVOKE_URL')).toBe(fixture.origins.oauthRevoke)
    expect(fixture.origins.oauthRevoke).toBe('https://auth.x.ai/oauth2/revoke')
  })

  it('T-R9-9b-grok lists invalid_tool_arguments in the frozen error taxonomy', () => {
    // R9-9 (N-4): the proxy answers a malformed tool call with this code, as the
    // Codex fixture already records.
    const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
      errorTaxonomy: unknown
    }
    const errors = fixture.errorTaxonomy
    // Witness: the taxonomy was read and is a non-empty list of codes.
    expect(Array.isArray(errors) && errors.length > 0).toBe(true)
    expect(errors).toContain('provider_unavailable')
    expect(errors).toContain('invalid_tool_arguments')
  })

  it('T-R10-3 lists context_length_exceeded in the frozen error taxonomy', () => {
    // R10 (M1): the proxy answers the upstream's context-window refusal with
    // this code, as the Codex fixture already records.
    const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
      errorTaxonomy: unknown
    }
    const errors = fixture.errorTaxonomy
    // Witness: the taxonomy was read and is a non-empty list of codes.
    expect(Array.isArray(errors) && errors.length > 0).toBe(true)
    expect(errors).toContain('provider_unavailable')
    expect(errors).toContain('context_length_exceeded')
  })
})