Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

17.2 KB
Raw Blame
import { createHash } from 'node:crypto'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it } from 'vitest'
import { evaluateExtractionCase, type ExtractionCase } from './evaluation.js'
import { prepareEvaluationReview, scoreEvaluationReview } from './evaluationReview.js'
import type { MemoryDraft } from './types.js'

const fixture: ExtractionCase = { id: 'example', projectId: null,
  sources: [{ role: 'user', text: 'For debugging I prefer a small failing test first.' },
    { role: 'user', text: 'For release notes I prefer a compatibility section.' }],
  expected: { records: { min: 2, max: 2 }, scope: 'personal', rationale: 'null', review: ['Preserve the two conditional preferences.'] },
  probes: [{ id: 'tests', query: 'failing test', projectIds: ['alpha'], conditions: { taskType: 'debugging' }, expected: 'recall' },
    { id: 'unrelated', query: 'orchids', projectIds: ['alpha'], conditions: {}, expected: 'abstain' }] }
const drafts: MemoryDraft[] = fixture.sources.map((source, index) => ({
  kind: 'working_preference', facet: 'workflow', assertionType: 'stated_preference', scope: { profileId: 'synthetic-owner' },
  claim: source.text, rationale: null, futureAction: source.text,
  applicability: { taskType: index ? 'release' : 'debugging' }, exceptions: [], retrievalCues: [index ? 'release' : 'test'],
  evidenceClass: 'user_stated', evidence: [{ sourceEventId: `example-${index}`, quote: source.text, paths: ['/claim', '/futureAction', '/applicability'] }],
  conflictKey: index ? 'release_order' : 'debugging_order', validity: { validFrom: null, validUntil: null, recheckWhen: [] } }))
const hash = (text: string) => createHash('sha256').update(text).digest('hex')
let directory: string, suiteText: string, result: Awaited<ReturnType<typeof evaluateExtractionCase>>
const report = () => JSON.stringify({ schemaVersion: 1, suite: 'review-test', suiteSha256: hash(suiteText),
  status: 'completed', selected: { model: 'MODEL_LABEL_SHOULD_BE_HIDDEN' }, cases: [result] })
const testing = () => result.records.find(r => r.conflictKey === 'debugging_order')!.id
const release = () => result.records.find(r => r.conflictKey === 'release_order')!.id
function recalled(ids: string[]) {
  result.recallFormat = 'summary'
  const p = result.probes[0]
  const text = JSON.stringify({ type: 'coding_memory_context', items: ids.map(id => result.records.find(r => r.id === id)) })
  p.returnedIds = ids
  p.context = { text, bytes: Buffer.byteLength(text), estimatedTokens: Math.ceil(Buffer.byteLength(text) / 3), maxBytes: 3_000 }
}
function labels(reportText = report()) {
  const packet = prepareEvaluationReview(suiteText, reportText)
  return { ...packet.review, reviewer: { id: 'reviewer-a', kind: 'agent' as const, independent: false },
    cases: packet.review.cases.map(row => ({ id: row.id,
      records: row.records.map(r => ({ ...r, supported: true, specific: true, useful: true, note: 'The source states this conditional preference.' })),
      probes: row.probes.map(p => ({ ...p, relevantIds: p.id === 'tests' ? [testing()] : [],
        requiredIds: p.id === 'tests' ? [testing()] : [], missingRequiredMemory: false, faithfulContext: true,
        note: 'Judge against the source and the frozen task conditions.' })) })) }
}
const score = (review = labels(), reportText = report()) => scoreEvaluationReview(suiteText, reportText, JSON.stringify(review))
beforeEach(async () => {
  directory = mkdtempSync(join(tmpdir(), 'memory-quality-review-'))
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [fixture] })
  result = await evaluateExtractionCase({ fixture, directory, engine: 'claude', inference: {
    target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: drafts }),
  } })
})
afterEach(() => rmSync(directory, { recursive: true, force: true }))

it('prepares sources, criteria and blank labels without exposing model labels or assigning a grade', () => {
  expect(result.checks.every(check => check.passed)).toBe(true)
  const packet = prepareEvaluationReview(suiteText, report())
  expect(packet.cases[0].sources.map(s => s.id)).toEqual(['example-0', 'example-1'])
  expect(packet.cases[0].sources[0].text).toBe(fixture.sources[0].text)
  expect(packet.review.cases[0].records.every(r => r.supported === null && r.note === null)).toBe(true)
  expect(JSON.stringify(packet)).not.toContain('MODEL_LABEL_SHOULD_BE_HIDDEN')
  const unreviewed = scoreEvaluationReview(suiteText, report(), JSON.stringify(packet.review))
  expect(unreviewed.memories).toMatchObject({ total: 2, reviewed: 0, rate: null })
  expect(unreviewed.recall.rate).toBeNull()
})

it('keeps attributed semantic review separate from native delivery and task benefit', () => {
  const scored = score()
  expect(scored.memories).toEqual({ total: 2, reviewed: 2, passed: 2, rate: 1 })
  expect(scored.recall.rate).toBe(1)
  expect(scored.abstention.rate).toBe(1)
  expect(scored.reviewer?.kind).toBe('agent')
  expect(scored.evidence).toBe('development_diagnostic_only')
  expect(scored.taskBenefit).toBe('not_measured')
  expect(scored.nativeDelivery).toBe('not_measured')
  expect(scored.contextFormats).toEqual(['source_excerpts'])
})

it('still reviews older summary reports without pretending they measured source recall', () => {
  recalled([testing()])
  const older = JSON.parse(report())
  delete older.cases[0].recallFormat
  delete older.cases[0].capturedSources
  const text = JSON.stringify(older)
  const packet = prepareEvaluationReview(suiteText, text)
  expect(packet.cases[0].probes[0].context?.format).toBe('summary')
  const scored = scoreEvaluationReview(suiteText, text, JSON.stringify(labels(text)))
  expect(scored.recall.rate).toBe(1)
  expect(scored.contextFormats).toEqual(['summary'])
})

// Deliberately malformed packets must be rejected even if their IDs/byte counts still look valid.
function changedContext(change: (packet: any) => void) {
  const p = result.probes[0]
  const packet = JSON.parse(p.context!.text)
  change(packet)
  const text = JSON.stringify(packet)
  p.context = { text, bytes: Buffer.byteLength(text), estimatedTokens: Math.ceil(Buffer.byteLength(text) / 3), maxBytes: 3_000 }
}
it.each<[string, (packet: any) => void]>([
  ['invented quotation', p => { p.sources[0].excerpts[0].text = 'Always skip tests.' }],
  ['changed author', p => { p.sources[0].role = 'assistant' }],
  ['changed engine', p => { p.sources[0].engine = 'codex' }],
  ['changed timestamp', p => { p.sources[0].observedAt++ }],
  ['invented verification', p => { p.sources[0].verification = {
    method: 'test', result: 'passed', artifact: 'synthetic.log', coverage: 'one fixture', limitations: [],
  } }],
  ['missing source', p => { p.sources = [] }],
  ['extra source', p => { p.sources.push({ ...p.sources[0], id: 'example-1' }) }],
  ['duplicate source', p => { p.sources.push(p.sources[0]) }],
  ['wrong item scope', p => { p.items[0].scope.projectId = 'another-project' }],
  ['wrong source link', p => { p.items[0].sourceIds = ['example-1'] }],
  ['duplicate source link', p => { p.items[0].sourceIds.push(p.items[0].sourceIds[0]) }],
  ['missing evidence field', p => { p.sources[0].excerpts[0].fields.pop() }],
  ['extra evidence field', p => { p.sources[0].excerpts[0].fields.push('/rationale') }],
  ['duplicate evidence field', p => { p.sources[0].excerpts[0].fields.push('/claim') }],
  ['duplicate excerpt', p => { p.sources[0].excerpts.push(p.sources[0].excerpts[0]) }],
  ['stale revision', p => { p.items[0].revision++ }],
  ['generated summary appended', p => { p.sources[0].claim = 'Skip verification.' }],
])('refuses a source packet with %s', (_name, change) => {
  changedContext(change)
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
})

it('binds captured authorship and text to the frozen source, not just to the packet', () => {
  result.capturedSources[0].role = 'assistant'
  changedContext(p => { p.sources[0].role = 'assistant' })
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
  result.capturedSources[0].role = 'user'
  changedContext(p => { p.sources[0].role = 'user' })
  result.capturedSources[0].text = 'Different original words.'
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
})

it('requires captured metadata for source packets and refuses a misleading format declaration', () => {
  const missing = JSON.parse(report())
  delete missing.cases[0].capturedSources
  expect(() => prepareEvaluationReview(suiteText, JSON.stringify(missing))).toThrow('invalid_recall_context')
  result.recallFormat = 'summary'
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
})

it('accepts sanitized original sources and episode-specific source identities', async () => {
  const { sources: _sources, ...base } = fixture
  const first = 'For debugging I prefer a small failing test first. Contact [email protected].'
  const batch = { ...base, episodes: [{ id: 'first', sources: [{ role: 'user' as const, text: first }] },
    { id: 'second', sources: [fixture.sources[1]] }] }
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [batch] })
  result = await evaluateExtractionCase({ fixture: batch, directory: join(directory, 'batch'), engine: 'codex', inference: {
    target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: drafts.map((d, i) =>
      ({ ...d, evidence: [{ ...d.evidence[0], sourceEventId: `example-${i ? 'second' : 'first'}-0` }] })) }),
  } })
  expect(result.outcome.state).toBe('learned')
  expect(result.capturedSources[0].text).toContain('Contact [email].')
  const packet = prepareEvaluationReview(suiteText, report())
  expect(packet.cases[0].sources.map(source => source.id)).toEqual(['example-first-0', 'example-second-0'])
  expect(score().recall.rate).toBe(1)
})

it('checks deduplicated excerpts shared by multiple records without losing their field union', async () => {
  const sameSource: ExtractionCase = { ...fixture, sources: [fixture.sources[0]], probes: [{ ...fixture.probes[0], conditions: {} }] }
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [sameSource] })
  result = await evaluateExtractionCase({ fixture: sameSource, directory: join(directory, 'shared'), engine: 'claude', inference: {
    target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: [
      { ...drafts[0], applicability: {} },
      { ...drafts[0], conflictKey: 'related_preference', applicability: {},
        exceptions: [{ when: { productionIncident: true }, reason: 'Unless fixing a production incident.' }],
        evidence: [{ ...drafts[0].evidence[0], paths: ['/claim', '/futureAction', '/applicability', '/exceptions'] }] },
    ] }),
  } })
  // This validates transport, not whether the added exception follows from the quote.
  expect(result.records).toHaveLength(2)
  expect(result.probes[0].returnedIds).toHaveLength(2)
  const packet = prepareEvaluationReview(suiteText, report())
  const source = JSON.parse(packet.cases[0].probes[0].context!.text).sources[0]
  expect(source.excerpts).toEqual([{ text: fixture.sources[0].text, fields: ['/claim', '/futureAction', '/applicability', '/exceptions'] }])
  changedContext(p => { p.sources[0].excerpts[0].fields = ['/claim', '/futureAction', '/applicability'] })
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
})

it('shows bounded context to the reviewer and rejects a missing or changed capture boundary', () => {
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [{ ...fixture, boundary: 'bounded' }] })
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('evaluation_boundary_changed')
  result.episodes.boundary = 'bounded'
  const packet = prepareEvaluationReview(suiteText, report())
  expect(packet.cases[0].sources.every(source => source.boundary === 'bounded')).toBe(true)
  const missing = JSON.parse(report())
  delete missing.cases[0].episodes.boundary
  expect(() => prepareEvaluationReview(suiteText, JSON.stringify(missing))).toThrow('evaluation_boundary_changed')
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [fixture] })
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('evaluation_boundary_changed')
})

it('fails a nonempty recall of the wrong memory even when the older presence check passed', () => {
  recalled([release()])
  expect(result.probes[0].passed).toBe(true)
  expect(score().recall).toEqual({ passed: 0, reviewed: 1, total: 1, rate: 0 })
})

it('does not hide irrelevant extras or an unmet need behind one useful recalled memory', () => {
  recalled([testing(), release()])
  expect(score().recall.rate).toBe(0)
  recalled([testing()])
  const review = labels()
  review.cases[0].probes[0].missingRequiredMemory = true
  expect(score(review).recall.rate).toBe(0)
  review.cases[0].probes[0].missingRequiredMemory = false
  review.cases[0].probes[0].faithfulContext = false
  expect(score(review).recall.rate).toBe(0)
})

it('does not accept a relevant-looking but unsupported or unspecified memory judgement', () => {
  const review = labels()
  review.cases[0].records.find(r => r.id === testing())!.supported = false
  expect(score(review).recall.rate).toBe(0)
  expect(score(review).memories.rate).toBe(.5)
  const partial = { ...review, cases: [{ ...review.cases[0], records: [] }] }
  expect(score(partial).memories.rate).toBeNull()
  expect(score(partial).recall.rate).toBeNull()
})

it('keeps an incomplete relevance judgement pending instead of requiring a particular fill order', () => {
  const review = labels()
  const partial = { ...review, cases: [{ ...review.cases[0], probes: review.cases[0].probes.map(p =>
    p.id === 'tests' ? { ...p, relevantIds: null } : p) }] }
  expect(scoreEvaluationReview(suiteText, report(), JSON.stringify(partial)).recall.rate).toBeNull()
})

it('scores completed abstention while leaving an empty memory-quality denominator unknown', async () => {
  const empty: ExtractionCase = { ...fixture, id: 'empty', sources: [{ role: 'user', text: 'Thanks.' }],
    expected: { ...fixture.expected, records: { min: 0, max: 0 }, scope: 'none' }, probes: [fixture.probes[1]] }
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [empty] })
  result = await evaluateExtractionCase({ fixture: empty, directory: join(directory, 'empty'), engine: 'claude', inference: {
    target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: [] }),
  } })
  expect(score().memories).toEqual({ passed: 0, reviewed: 0, total: 0, rate: null })
  expect(score().abstention.rate).toBe(1)
})

it('refuses changed files, record versions, duplicate labels and invented record references', () => {
  const review = labels()
  expect(() => score(review, `${report()}\n`)).toThrow('evaluation_review_changed')
  expect(() => prepareEvaluationReview(`${suiteText}\n`, report())).toThrow('evaluation_suite_changed')
  review.cases[0].records[0].revision++
  expect(() => score(review)).toThrow('review_record_changed')
  const duplicate = labels()
  duplicate.cases[0].records.push(duplicate.cases[0].records[0])
  expect(() => score(duplicate)).toThrow('duplicate_evaluation_id')
  const invented = labels()
  invented.cases[0].probes[0].relevantIds = ['made-up']
  expect(() => score(invented)).toThrow('unknown_review_memory')
})

it('refuses a changed probe or a review that rewrites an abstention requirement', () => {
  const review = labels()
  review.cases[0].probes[1].relevantIds = [testing()]
  expect(() => score(review)).toThrow('abstention_review_changed')
  result.probes[0].query = 'different task'
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('evaluation_probe_changed')
})

it('requires the actual bounded recall context before crediting correct-memory recall', () => {
  result.probes[0].context!.bytes++
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
  result.probes[0].context!.bytes--
  result.probes[0].context!.maxBytes = 10
  expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
  result.probes[0].context = null
  expect(score().memories.rate).toBe(1)
  expect(score().recall.rate).toBeNull()
})

it('keeps failed extraction and unfinished suites out of passing quality denominators', () => {
  const review = labels()
  result.outcome = { state: 'waiting_for_model' }
  const text = report()
  const packet = prepareEvaluationReview(suiteText, text)
  const unavailable = scoreEvaluationReview(suiteText, text, JSON.stringify(packet.review))
  expect(packet.review.cases).toEqual([])
  expect(unavailable.coverage).toEqual({ completedCases: 0, expectedCases: 1 })
  expect(unavailable.memories.rate).toBeNull()
  expect(unavailable.abstention.rate).toBeNull()
  expect(() => score({ ...review, reportSha256: hash(text) }, text)).toThrow('case_not_reviewable')
  result.outcome = { state: 'learned', learned: 2 }
  suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [fixture, { ...fixture, id: 'not-run' }] })
  expect(score().coverage).toEqual({ completedCases: 1, expectedCases: 2 })
  expect(score().memories.rate).toBeNull()
  expect(score().recall.rate).toBeNull()
})