Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test: require the catalog context window end to end (#731)
Alfredo Lopez1174a7b433cd
diff --git a/codex-llm-proxy/test/catalogContextWindow.test.ts b/codex-llm-proxy/test/catalogContextWindow.test.tsnew file mode 100644index 000000000..3e2d742bb--- /dev/null+++ b/codex-llm-proxy/test/catalogContextWindow.test.ts@@ -0,0 +1,61 @@+import { describe, expect, it } from 'vitest'+import { readFileSync } from 'node:fs'+import { listCodexModels } from '../src/codexTransport.js'++const lookup = async () => [{ address: '1.2.3.4', family: 4 }]++function jsonResponse(body: unknown): Response {+ return new Response(JSON.stringify(body), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })+}++/** A recorded upstream catalog response, sanitized (see its `_provenance`). */+const recorded = JSON.parse(+ readFileSync(new URL('./fixtures/codexModelsCatalog.sanitized.json', import.meta.url), 'utf-8')+) as { models: Array<{ slug: string; context_window: number; max_context_window: number }> }++describe('Codex catalog context window (#731 R3-4)', () => {+ it('T-R3-4a keeps the recorded context_window as contextWindowTokens', async () => {+ // The fixture must carry the case that tells the two fields apart.+ const sol = recorded.models.find(row => row.slug === 'gpt-5.6-sol')+ expect(sol?.max_context_window).toBeGreaterThan(sol!.context_window)++ const listed = await listCodexModels({+ accessToken: 'tok',+ fetchFn: (async () => jsonResponse(recorded)) as typeof fetch,+ lookup,+ })+ expect(listed.outcome).toBe('ready')+ expect(listed.models.map(row => [row.model, row.contextWindowTokens])).toEqual([+ ['gpt-5.6-sol', 272_000],+ ['gpt-5.5', 272_000],+ ])+ })++ it('T-R3-4b omits a window that is not a positive integer within the stored column', async () => {+ // control-api stores the window in a Postgres INTEGER column; a larger+ // value would fail the whole catalog sync, not just this row.+ const invalid = [0, -1, 1.5, '272000', null, 2_147_483_648]+ const listed = await listCodexModels({+ accessToken: 'tok',+ fetchFn: (async () =>+ jsonResponse({+ models: [+ ...invalid.map((contextWindow, index) => ({+ slug: `model-${index}`,+ context_window: contextWindow,+ })),+ { slug: 'model-max', context_window: 2_147_483_647 },+ ],+ })) as typeof fetch,+ lookup,+ })+ // Witness: every row was listed, so the omission is per field, not per row.+ expect(listed.models).toHaveLength(invalid.length + 1)+ expect(listed.models.filter(row => 'contextWindowTokens' in row)).toEqual([+ { model: 'model-max', contextWindowTokens: 2_147_483_647 },+ ])+ })+})diff --git a/codex-llm-proxy/test/fixtures/codexModelsCatalog.sanitized.json b/codex-llm-proxy/test/fixtures/codexModelsCatalog.sanitized.jsonnew file mode 100644index 000000000..18736ab87--- /dev/null+++ b/codex-llm-proxy/test/fixtures/codexModelsCatalog.sanitized.json@@ -0,0 +1,22 @@+{+ "_provenance": {+ "origin": "https://chatgpt.com/backend-api/codex/models",+ "recordedBy": "Codex CLI 0.155.0-alpha.16 models cache",+ "fetchedAt": "2026-09-22T20:16:19.046Z",+ "sanitization": "Two of ten rows kept; each row reduced to its identity, display name and context-window fields. The envelope is the upstream one ({ models: [...] })."+ },+ "models": [+ {+ "slug": "gpt-5.6-sol",+ "display_name": "GPT-5.6-Sol",+ "context_window": 272000,+ "max_context_window": 872000+ },+ {+ "slug": "gpt-5.5",+ "display_name": "GPT-5.5",+ "context_window": 272000,+ "max_context_window": 272000+ }+ ]+}diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 716800e60..026eff3ee 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex a4c6f4544..8ec051e38 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.601",+ "version": "0.1.602","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts b/control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.tsindex 86e475fe1..f8e8a0cb9 100644--- a/control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts+++ b/control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts@@ -3,7 +3,10 @@ import { createHash, randomBytes } from 'node:crypto'import { Pool } from 'pg'import { initDb } from '../src/db.js'import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'-import { syncCodexSubscriptionCatalog } from '../src/services/codexSubscriptionCatalog.js'+import {+ type CodexDiscoveredModel,+ syncCodexSubscriptionCatalog,+} from '../src/services/codexSubscriptionCatalog.js'import { insertInitialCodexSubscriptionConnection } from '../src/services/codexSubscriptionConnection.js'const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL@@ -92,4 +95,47 @@ describeRealPostgres('Codex subscription catalog on real PostgreSQL', () => {)expect(kept.rows[0]?.stale).toBe(false)})++ it('T-R3-4c stores the catalog window and refreshes it on an existing row (#731 R3-4)', async () => {+ const model = 'gpt-5.5'+ const sync = (models: CodexDiscoveredModel[]) =>+ syncCodexSubscriptionCatalog(+ pool,+ {+ async listModels() {+ return { outcome: 'ready', models }+ },+ },+ 'access-token'+ )+ const windows = async () => {+ const connection = await pool.query<{ context_window_tokens: number | null }>(+ `SELECT context_window_tokens FROM codex_catalog_models WHERE model = $1`,+ [model]+ )+ const union = await pool.query<{ context_window_tokens: number | null }>(+ `SELECT context_window_tokens FROM llm_allowed_models+ WHERE provider = 'codex-subscription' AND model = $1`,+ [model]+ )+ return {+ connection: connection.rows.map(row => row.context_window_tokens),+ union: union.rows.map(row => row.context_window_tokens),+ }+ }++ // Discovered before the catalog supplied a window.+ expect((await sync([{ model }])).added).toBe(1)+ expect(await windows()).toEqual({ connection: [null], union: [null] })++ // The catalog now supplies one; the existing rows take the refresh path.+ const supplied = await sync([{ model, contextWindowTokens: 272_000 }])+ expect(supplied.refreshed).toBe(1)+ expect(await windows()).toEqual({ connection: [272_000], union: [272_000] })++ // A later catalog without a window keeps the stored value.+ const silent = await sync([{ model }])+ expect(silent.refreshed).toBe(1)+ expect(await windows()).toEqual({ connection: [272_000], union: [272_000] })+ })})diff --git a/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts b/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.tsindex 89e71dc77..5aefaaca1 100644--- a/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts+++ b/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts@@ -5,6 +5,7 @@ import { type DbClient, initDb } from '../src/db.js'import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'import {type GrokCatalogTransport,+ type GrokDiscoveredModel,type GrokTransactionRunner,syncGrokSubscriptionCatalog,} from '../src/services/grokSubscriptionCatalog.js'@@ -544,4 +545,53 @@ describeRealPostgres('Grok subscription connection on real PostgreSQL', () => {)).rejects.toBeInstanceOf(GrokSubscriptionConnectionKeyConflictError)})++ it('T-R3-4c-grok stores the catalog window and refreshes it on an existing row (#731 R3-4)', async () => {+ const key = 'team-grok-window'+ const model = 'grok-window-r34'+ const created = await insertInitialGrokSubscriptionConnection(+ pool,+ KEY,+ { refreshToken: 'refresh-window', accessToken: 'access', accountFingerprint: 'fp-window' },+ key+ )+ const sync = (models: GrokDiscoveredModel[]) =>+ syncGrokSubscriptionCatalog(+ pool,+ { listModels: async () => ({ outcome: 'ready', models }) },+ 'access',+ { connectionKey: key },+ { withTransaction: poolTransaction }+ )+ const windows = async () => {+ const connection = await pool.query<{ context_window_tokens: number | null }>(+ `SELECT context_window_tokens FROM grok_catalog_models+ WHERE connection_id = $1 AND model = $2`,+ [created.id, model]+ )+ const union = await pool.query<{ context_window_tokens: number | null }>(+ `SELECT context_window_tokens FROM llm_allowed_models+ WHERE provider = 'grok-subscription' AND model = $1`,+ [model]+ )+ return {+ connection: connection.rows.map(row => row.context_window_tokens),+ union: union.rows.map(row => row.context_window_tokens),+ }+ }++ // Discovered before the catalog supplied a window.+ expect((await sync([{ model }])).added).toBe(1)+ expect(await windows()).toEqual({ connection: [null], union: [null] })++ // The catalog now supplies one; the existing rows take the refresh path.+ const supplied = await sync([{ model, contextWindowTokens: 500_000 }])+ expect(supplied.refreshed).toBe(1)+ expect(await windows()).toEqual({ connection: [500_000], union: [500_000] })++ // A later catalog without a window keeps the stored value.+ const silent = await sync([{ model }])+ expect(silent.refreshed).toBe(1)+ expect(await windows()).toEqual({ connection: [500_000], union: [500_000] })+ })})diff --git a/grok-llm-proxy/test/catalogContextWindow.test.ts b/grok-llm-proxy/test/catalogContextWindow.test.tsnew file mode 100644index 000000000..171191cc5--- /dev/null+++ b/grok-llm-proxy/test/catalogContextWindow.test.ts@@ -0,0 +1,56 @@+import { describe, expect, it } from 'vitest'+import { readFileSync } from 'node:fs'+import { listGrokModels } from '../src/grokTransport.js'++const lookup = async () => [{ address: '1.2.3.4', family: 4 }]++function jsonResponse(body: unknown): Response {+ return new Response(JSON.stringify(body), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })+}++/** A recorded upstream catalog row, sanitized (see its `_provenance`). */+const recorded = JSON.parse(+ readFileSync(new URL('./fixtures/grokModelsCatalog.sanitized.json', import.meta.url), 'utf-8')+) as { data: Array<{ id: string; context_window: number }> }++describe('Grok catalog context window (#731 R3-4)', () => {+ it('T-R3-4a-grok keeps the recorded context_window as contextWindowTokens', async () => {+ const listed = await listGrokModels({+ accessToken: 'tok',+ fetchFn: (async () => jsonResponse(recorded)) as typeof fetch,+ lookup,+ })+ expect(listed.outcome).toBe('ready')+ expect(listed.models.map(row => [row.model, row.contextWindowTokens])).toEqual([+ ['grok-4.6', 500_000],+ ])+ })++ it('T-R3-4b-grok omits a window that is not a positive integer within the stored column', async () => {+ // control-api stores the window in a Postgres INTEGER column; a larger+ // value would fail the whole catalog sync, not just this row.+ const invalid = [0, -1, 1.5, '500000', null, 2_147_483_648]+ const listed = await listGrokModels({+ accessToken: 'tok',+ fetchFn: (async () =>+ jsonResponse({+ data: [+ ...invalid.map((contextWindow, index) => ({+ id: `model-${index}`,+ context_window: contextWindow,+ })),+ { id: 'model-max', context_window: 2_147_483_647 },+ ],+ })) as typeof fetch,+ lookup,+ })+ // Witness: every row was listed, so the omission is per field, not per row.+ expect(listed.models).toHaveLength(invalid.length + 1)+ expect(listed.models.filter(row => 'contextWindowTokens' in row)).toEqual([+ { model: 'model-max', contextWindowTokens: 2_147_483_647 },+ ])+ })+})diff --git a/grok-llm-proxy/test/fixtures/grokModelsCatalog.sanitized.json b/grok-llm-proxy/test/fixtures/grokModelsCatalog.sanitized.jsonnew file mode 100644index 000000000..0e1f07502--- /dev/null+++ b/grok-llm-proxy/test/fixtures/grokModelsCatalog.sanitized.json@@ -0,0 +1,16 @@+{+ "_provenance": {+ "origin": "https://cli-chat-proxy.grok.com/v1/models",+ "recordedBy": "Grok CLI 1.0.30 models cache",+ "fetchedAt": "2026-09-16T15:05:42.026248Z",+ "sanitization": "The per-model object is kept as recorded, reduced to its identity, name and context-window fields; the CLI's own key and endpoint settings are dropped. The CLI re-wraps the list, so the OpenAI-compatible { data: [...] } envelope is reconstructed. The field name is confirmed by the live catalog sync in #731 R3-5."+ },+ "data": [+ {+ "id": "grok-4.6",+ "model": "grok-4.6",+ "name": "Grok 4.6",+ "context_window": 500000+ }+ ]+}diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 65ec0a513..0296e00b9 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.395",+ "version": "0.1.396","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.395",+ "version": "0.1.396","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 60d10aafc..7aa7cda51 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.395",+ "version": "0.1.396","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/agent/__tests__/taskExecutor.test.ts b/mcp-host/src/agent/__tests__/taskExecutor.test.tsindex e469d0c2f..a10ada663 100644--- a/mcp-host/src/agent/__tests__/taskExecutor.test.ts+++ b/mcp-host/src/agent/__tests__/taskExecutor.test.ts@@ -2340,6 +2340,91 @@ describe('TaskExecutor context manager message bound (#731)', () => {})})+describe('TaskExecutor subscription context window (#731 R3-4)', () => {+ beforeEach(() => {+ vi.mocked(runToolUseLoop).mockReset()+ })++ // ~150k tokens by the byte heuristic the subscription counters use: above 0.8+ // of a 100k window, below 0.8 of the 256k subscription default.+ function largeHistory(): ChatMessage[] {+ const msgs: ChatMessage[] = [{ role: 'system', content: 'sys' }]+ for (let i = 0; i < 60; i++) {+ msgs.push({ role: i % 2 === 0 ? 'user' : 'assistant', content: 'word '.repeat(2_000) })+ }+ return msgs+ }++ function depsFor(providerType: string, contextWindowTokens?: number) {+ return createDeps({+ modelName: 'gpt-5.5',+ contextWindowTokens,+ llmProvider: {+ completeSingleTurn: vi.fn(),+ completeSingleTurnWithTools: vi.fn(),+ getProviderType: () => providerType,+ } as any,+ })+ }++ // Runs one task and hands back the context manager the loop received.+ async function loopContextManager(deps: TaskExecutorDeps) {+ vi.mocked(runToolUseLoop).mockResolvedValueOnce({ type: 'response', content: 'ok' } as any)+ await new TaskExecutor(createTask('hi'), deps).run()+ const loopConfig = vi.mocked(runToolUseLoop).mock.calls.at(-1)?.[0]+ if (!loopConfig) throw new Error('Expected runToolUseLoop to receive a loop config')+ return loopConfig.contextManager+ }++ it('T-R3-4e a codex-subscription task without a catalog window runs with the 256k default', async () => {+ const msgs = largeHistory()+ const manager = await loopContextManager(depsFor('codex-subscription'))+ expect(manager).toBeInstanceOf(PressureContextManager)+ expect(await manager.manage(msgs, makeFakeConversation())).toBe(msgs)+ // Witness: under a 100k catalog window the same history is compacted, so the+ // passthrough above is the window's doing.+ const narrow = await loopContextManager(depsFor('codex-subscription', 100_000))+ expect((await narrow.manage(msgs, makeFakeConversation())).length).toBeLessThan(msgs.length)+ })++ it('T-R3-4f logs the window and its source once per subscription task', async () => {+ const info = vi.spyOn(logger, 'info').mockImplementation(() => {})+ try {+ await loopContextManager(depsFor('codex-subscription'))+ await loopContextManager(depsFor('grok-subscription', 500_000))+ await loopContextManager(depsFor('openai'))+ const resolved = info.mock.calls+ .map(call => call[0] as Record<string, unknown>)+ .filter(fields => fields?.event === 'context_window_resolved')+ expect(resolved).toEqual([+ {+ event: 'context_window_resolved',+ component: 'TaskExecutor',+ taskId: expect.any(String),+ provider: 'codex-subscription',+ model: 'gpt-5.5',+ contextWindowTokens: 256_000,+ source: 'default',+ },+ {+ event: 'context_window_resolved',+ component: 'TaskExecutor',+ taskId: expect.any(String),+ provider: 'grok-subscription',+ model: 'gpt-5.5',+ contextWindowTokens: 500_000,+ source: 'catalog',+ },+ ])+ // Witness: all three tasks reached the loop, so the openai task logged+ // nothing because it has no subscription window, not because it never ran.+ expect(runToolUseLoop).toHaveBeenCalledTimes(3)+ } finally {+ info.mockRestore()+ }+ })+})+describe('TaskExecutor history compaction threshold follows the context window (#731)', () => {// ~100k tokens by either count (tiktoken reads one token per `word `, the// byte heuristic 125k): above the 80k literal default, below 0.8 of a 1Mdiff --git a/mcp-host/src/llm/__tests__/registry.test.ts b/mcp-host/src/llm/__tests__/registry.test.tsindex ffe6ab5d4..30a7e6665 100644--- a/mcp-host/src/llm/__tests__/registry.test.ts+++ b/mcp-host/src/llm/__tests__/registry.test.ts@@ -10,7 +10,14 @@ import { LIMITS as GROK_LIMITS } from '@clerum/grok-provider-attempt-contract'import { LIMITS as CODEX_LIMITS } from '@clerum/llm-provider-attempt-contract'import { apiKeysFromEnv, createLLMProvider } from '../index'import { makeProvider } from '../registry'-import { ALL_PROVIDERS, descriptorFor, isLlmProvider, primarySlot } from '../registryCore'+import {+ ALL_PROVIDERS,+ SUBSCRIPTION_DEFAULT_CONTEXT_WINDOW_TOKENS,+ descriptorFor,+ isLlmProvider,+ primarySlot,+ resolveContextWindow,+} from '../registryCore'describe('provider registry — auto-detection order (§5.9)', () => {it('preserves the dev priority prefix openai > claude > zai > bailian > vertex > bedrock', () => {@@ -261,3 +268,36 @@ describe('provider registry — contract message bound (#731)', () => {expect(descriptorFor('openai').maxMessages).toBeUndefined()})})++describe('provider registry — context window (#731 R3-4)', () => {+ it('T-R3-4d subscription providers default to a 256k window when their catalog names none', () => {+ expect(SUBSCRIPTION_DEFAULT_CONTEXT_WINDOW_TOKENS).toBe(256_000)+ expect(descriptorFor('codex-subscription').defaultContextWindowTokens).toBe(256_000)+ expect(descriptorFor('grok-subscription').defaultContextWindowTokens).toBe(256_000)+ // Other providers keep CLERUM_CONTEXT_MAX_TOKENS.+ expect(descriptorFor('openai').defaultContextWindowTokens).toBeUndefined()+ })++ it('T-R3-4d resolves the catalog window first, then the subscription default, then the env value', () => {+ expect(resolveContextWindow('codex-subscription', 272_000, 100_000)).toEqual({+ contextWindowTokens: 272_000,+ source: 'catalog',+ })+ expect(resolveContextWindow('grok-subscription', undefined, 100_000)).toEqual({+ contextWindowTokens: 256_000,+ source: 'default',+ })+ expect(resolveContextWindow('openai', 400_000, 100_000)).toEqual({+ contextWindowTokens: 400_000,+ source: 'catalog',+ })+ expect(resolveContextWindow('openai', undefined, 100_000)).toEqual({+ contextWindowTokens: 100_000,+ source: 'env',+ })+ expect(resolveContextWindow('unregistered', undefined, 100_000)).toEqual({+ contextWindowTokens: 100_000,+ source: 'env',+ })+ })+})