Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

10.3 KB
Raw Blame
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { evaluateExtractionCase, type ExtractionCase, type ExtractionBatchCase } from './evaluation.js'
import { nativeMemoryUsage } from './inferenceProcess.js'
import type { MemoryDraft } from './types.js'

let directory: string
beforeEach(() => { directory = mkdtempSync(join(tmpdir(), 'memory-evaluation-')) })
afterEach(() => { rmSync(directory, { recursive: true, force: true }) })
const fixture: ExtractionCase = {
  id: 'synthetic', projectId: null,
  sources: [{ role: 'user', text: 'For debugging I prefer a small failing test first.' }],
  expected: { records: { min: 1, max: 1 }, scope: 'personal', rationale: 'null', review: ['SECRET_EXPECTATION_NOT_IN_PROMPT'] },
  probes: [{ id: 'expected', query: 'failing test', projectIds: ['alpha'], conditions: { taskType: 'debugging' }, expected: 'recall' },
    { id: 'negative', query: 'failing test', projectIds: ['alpha'], conditions: {}, expected: 'abstain' }],
}
const draft: MemoryDraft = {
  kind: 'working_preference', facet: 'debugging', assertionType: 'stated_preference', scope: { profileId: 'synthetic-owner' },
  claim: 'For debugging prefer a small failing test first.', rationale: null, futureAction: 'Start with a small failing test.',
  applicability: { taskType: 'debugging' }, exceptions: [], retrievalCues: ['test'], evidenceClass: 'user_stated',
  evidence: [{ sourceEventId: 'synthetic-0', quote: fixture.sources[0].text, paths: ['/claim', '/futureAction', '/applicability'] }],
  conflictKey: 'debugging_order', validity: { validFrom: null, validUntil: null, recheckWhen: [] },
}
function provider(proposals: MemoryDraft[]) {
  return { target: async () => ({ state: 'ready' as const, key: 'synthetic-context' }), run: vi.fn(async () => JSON.stringify({ proposals })) }
}

it('runs real admission and scoped recall without leaking its rubric or probes to inference', async () => {
  const inference = provider([draft])
  const result = await evaluateExtractionCase({ fixture, directory, engine: 'claude', inference })
  expect(result.outcome).toEqual({ state: 'learned', learned: 1 })
  expect(result.checks.every(check => check.passed)).toBe(true)
  expect(result.semanticReview.status).toBe('pending')
  expect(result.recallFormat).toBe('source_excerpts')
  const packet = JSON.parse(result.probes[0].context!.text)
  expect(packet.type).toBe('coding_memory_sources')
  expect(packet.sources[0]).toMatchObject({ id: 'synthetic-0', engine: 'claude', role: 'user',
    excerpts: [{ text: fixture.sources[0].text, fields: draft.evidence[0].paths }] })
  expect(result.probes[0].context!.text).not.toContain(draft.claim)
  expect(result.probes[0].context!.text).not.toContain(draft.futureAction)
  const prompt = (inference.run.mock.calls[0] as unknown as [string])[0]
  expect(prompt).toContain(fixture.sources[0].text)
  expect(prompt).not.toContain('SECRET_EXPECTATION_NOT_IN_PROMPT')
  expect(prompt).not.toContain('"id":"negative"')
})

it('retains sanitized captured source metadata even when extraction produces no memory', async () => {
  const result = await evaluateExtractionCase({ fixture: { ...fixture,
    sources: [{ role: 'user', text: ' Contact [email protected] about debugging. ' }] },
    directory, engine: 'codex', inference: provider([]) })
  expect(result.capturedSources).toEqual([expect.objectContaining({ id: 'synthetic-0', role: 'user',
    engine: 'codex', projectId: null, text: ' Contact [email] about debugging. ', observedAt: expect.any(Number) })])
  expect(JSON.stringify(result)).not.toContain('[email protected]')
})

it('keeps the older summary packet available as an explicit comparison', async () => {
  const result = await evaluateExtractionCase({ fixture, directory, engine: 'claude',
    inference: provider([draft]), recallFormat: 'summary' })
  expect(result.recallFormat).toBe('summary')
  expect(JSON.parse(result.probes[0].context!.text).type).toBe('coding_memory_context')
  expect(result.probes[0].context!.text).toContain(draft.claim)
})

it('rejects an unknown recall format before inference', async () => {
  const inference = provider([draft])
  await expect(evaluateExtractionCase({ fixture, directory, engine: 'claude', inference,
    recallFormat: 'unknown' as 'summary' })).rejects.toThrow('invalid_evaluation_recall_format')
  expect(inference.run).not.toHaveBeenCalled()
})

it('exposes independently specified condition mismatch instead of copying generated keys into the probe', async () => {
  const result = await evaluateExtractionCase({ fixture, directory, engine: 'codex',
    inference: provider([{ ...draft, applicability: { task: 'debugging' } }]) })
  expect(result.outcome.state).toBe('learned')
  expect(result.checks.find(check => check.name === 'recall:expected')?.passed).toBe(false)
})

it('never scores invalid output as successful abstention', async () => {
  const empty: ExtractionCase = { ...fixture, expected: { ...fixture.expected, records: { min: 0, max: 0 }, scope: 'none' }, probes: [] }
  const inference = provider([])
  inference.run = vi.fn(async () => 'not json')
  const result = await evaluateExtractionCase({ fixture: empty, directory, engine: 'claude', inference })
  expect(result.outcome.state).toBe('failed')
  expect(result.records).toEqual([])
  expect(result.checks.find(check => check.name === 'completed_extraction')?.passed).toBe(false)
  expect(result.checks.find(check => check.name === 'record_count')?.passed).toBeNull()
  expect(result.semanticReview.status).toBe('not_reviewable')
})

it('marks recall probes inconclusive when the selected provider is unavailable', async () => {
  const result = await evaluateExtractionCase({ fixture, directory, engine: 'claude', inference: {
    target: async () => ({ state: 'ready', key: 'synthetic-context' }), run: async () => null,
  } })
  expect(result.probes.every(probe => probe.status === 'not_run' && probe.passed === null)).toBe(true)
  expect(result.checks.filter(check => check.name !== 'completed_extraction').every(check => check.passed === null)).toBe(true)
})

it('evaluates separate sessions together without leaking their rubric, and reports each episode outcome', async () => {
  const { sources, ...base } = fixture
  const batch: ExtractionBatchCase = { ...base, episodes: [{ id: 'preference', sources },
    { id: 'unrelated', sources: [{ role: 'assistant', text: 'I used MongoDB for a throwaway experiment.' }, { role: 'user', text: 'Thanks.' }] }] }
  const inference = provider([{ ...draft, evidence: [{ ...draft.evidence[0], sourceEventId: 'synthetic-preference-0' }] }])
  const result = await evaluateExtractionCase({ fixture: batch, directory, engine: 'claude', inference })
  expect(result.checks.every(check => check.passed)).toBe(true)
  expect(result.episodes).toEqual({ expected: 2, reviewed: 2, jobs: { learned: 1, no_useful_memory: 1 }, boundary: 'complete' })
  expect(result.semanticReview.status).toBe('pending')
  expect(inference.run).toHaveBeenCalledOnce()
  const prompt = (inference.run.mock.calls[0] as unknown as [string])[0]
  expect(prompt).not.toContain('SECRET_EXPECTATION_NOT_IN_PROMPT')
  expect(prompt).not.toContain('"id":"negative"')
  expect(prompt).toContain('synthetic-synthetic-preference')
  expect(prompt).toContain('synthetic-synthetic-unrelated')
})

it('does not score a partial batch as successful abstention when input limits leave episodes queued', async () => {
  const { sources: _sources, ...base } = fixture
  const batch: ExtractionBatchCase = { ...base, expected: { ...base.expected, records: { min: 0, max: 0 }, scope: 'none' },
    episodes: Array.from({ length: 4 }, (_, index) => ({ id: `large-${index}`, sources: [{ role: 'user', text: 'x'.repeat(32_000) }] })) }
  const result = await evaluateExtractionCase({ fixture: batch, directory, engine: 'claude', inference: provider([]) })
  expect(result.outcome.state).toBe('no_useful_memory')
  expect(result.episodes).toEqual({ expected: 4, reviewed: 2, jobs: { no_useful_memory: 2, queued: 2 }, boundary: 'complete' })
  expect(result.checks[0]).toEqual({ name: 'completed_extraction', passed: false })
  expect(result.checks.slice(1).every(check => check.passed === null)).toBe(true)
  expect(result.probes.every(probe => probe.status === 'not_run')).toBe(true)
  expect(result.semanticReview.status).toBe('not_reviewable')
})

it('rejects an oversized diagnostic group before invoking inference', async () => {
  const { sources, ...base } = fixture
  const inference = provider([])
  const batch: ExtractionBatchCase = { ...base, episodes: Array.from({ length: 5 }, (_, index) => ({ id: `case-${index}`, sources })) }
  await expect(evaluateExtractionCase({ fixture: batch, directory, engine: 'claude', inference })).rejects.toThrow('invalid_evaluation_episodes')
  expect(inference.run).not.toHaveBeenCalled()
})

it('runs bounded excerpts under the real queue restriction and exposes the boundary in its report', async () => {
  const inferred: MemoryDraft = { ...draft, assertionType: 'observed_usage', evidenceClass: 'inferred' }
  const complete = await evaluateExtractionCase({ fixture, directory, engine: 'codex', inference: provider([inferred]) })
  expect(complete.outcome).toEqual({ state: 'learned', learned: 1 })
  const inference = provider([inferred])
  const bounded = await evaluateExtractionCase({ fixture: { ...fixture, boundary: 'bounded' },
    directory: join(directory, 'bounded'), engine: 'codex', inference })
  expect(bounded.outcome).toEqual({ state: 'failed', reason: 'bounded_context_evidence' })
  expect(bounded.episodes.boundary).toBe('bounded')
  expect(bounded.records).toEqual([])
  expect(bounded.probes.every(probe => probe.status === 'not_run')).toBe(true)
  expect((inference.run.mock.calls[0] as unknown as [string])[0]).toContain('"context":"bounded"')
})

it('rejects an unrecognized boundary before calling the model', async () => {
  const inference = provider([])
  await expect(evaluateExtractionCase({ fixture: { ...fixture, boundary: 'partial' as 'bounded' }, directory, engine: 'codex', inference }))
    .rejects.toThrow('invalid_evaluation_boundary')
  expect(inference.run).not.toHaveBeenCalled()
})

it.each([undefined, {}, { input_tokens: 1 }, { input_tokens: -1, output_tokens: 2 },
  { input_tokens: 1.1, output_tokens: 2 }, { input_tokens: 2, output_tokens: '3' }])('keeps unknown/invalid native usage unknown', value => {
  expect(nativeMemoryUsage(value)).toBeUndefined()
})