Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

13.1 KB
Raw Blame
/** Offline semantic review of native diagnostic output. Never invokes a model or changes memory. */
import { createHash } from 'node:crypto'
import { z } from 'zod'
import { canonical } from './admission.js'
import { conditionsSchema, draftSchema, MemoryError, sourceSchema } from './types.js'
import { reviewRecallContext } from './evaluationContext.js'

const id = z.string().min(1).max(200).regex(/^[A-Za-z0-9_.:-]+$/)
const hash = z.string().regex(/^[a-f0-9]{64}$/)
const count = z.number().int().nonnegative().safe()
const source = z.object({ role: z.enum(['user', 'assistant', 'tool', 'reference', 'derived']), text: z.string().min(1).max(32_000) })
const probe = z.object({ id, query: z.string().max(4_000), projectIds: z.array(id), conditions: conditionsSchema,
  expected: z.enum(['recall', 'abstain']) })
const fixture = z.object({ id, projectId: id.nullable(), sources: z.array(source).min(1).optional(),
  boundary: z.enum(['complete', 'bounded']).optional(),
  episodes: z.array(z.object({ id, sources: z.array(source).min(1) })).min(1).max(4).optional(),
  expected: z.object({ review: z.array(z.string().max(2_000)) }), probes: z.array(probe) })
  .refine(row => !!row.sources !== !!row.episodes, 'choose sources or episodes')
const suiteSchema = z.object({ schemaVersion: z.literal(1), suite: id, cases: z.array(fixture).min(1).max(100) })
const recordSchema = draftSchema.safeExtend({ schemaVersion: z.literal(1), id, revision: count.positive(),
  state: z.enum(['active', 'tentative', 'needs_verification', 'superseded', 'archived']), createdAt: count, updatedAt: count })
const contextSchema = z.object({ text: z.string().max(16_000), bytes: count, maxBytes: count.max(16_000), estimatedTokens: count })
const resultSchema = z.object({ id, outcome: z.object({ state: z.string() }),
  recallFormat: z.enum(['summary', 'source_excerpts']).optional(), capturedSources: z.array(sourceSchema).max(512).optional(),
  episodes: z.object({ expected: count, reviewed: count, jobs: z.record(z.string(), count),
    boundary: z.enum(['complete', 'bounded']).optional() }).optional(),
  checks: z.array(z.object({ name: z.string(), passed: z.boolean().nullable() })),
  records: z.array(recordSchema).max(100),
  probes: z.array(probe.extend({ status: z.string(), returnedIds: z.array(id).max(6), passed: z.boolean().nullable(),
    context: contextSchema.nullable().optional() })) })
const reportSchema = z.object({ schemaVersion: z.literal(1), suite: id, suiteSha256: hash,
  status: z.enum(['running', 'stopped', 'completed']), cases: z.array(resultSchema).max(100) })
const note = z.string().trim().min(1).max(2_000).nullable()
const recordReview = z.object({ id, revision: count.positive(), supported: z.boolean().nullable(),
  specific: z.boolean().nullable(), useful: z.boolean().nullable(), note }).strict()
const probeReview = z.object({ id, relevantIds: z.array(id).max(100).nullable(),
  requiredIds: z.array(id).max(100).nullable(), missingRequiredMemory: z.boolean().nullable(),
  faithfulContext: z.boolean().nullable(), note }).strict()
const reviewSchema = z.object({ schemaVersion: z.literal(1), reportSha256: hash, suiteSha256: hash,
  reviewer: z.object({ id, kind: z.enum(['human', 'agent']), independent: z.boolean() }).strict().nullable(),
  cases: z.array(z.object({ id, records: z.array(recordReview).max(100), probes: z.array(probeReview).max(100) }).strict()).max(100),
}).strict()
const sha256 = (text: string) => createHash('sha256').update(text).digest('hex')
function decode<T>(schema: z.ZodType<T>, text: string): T {
  try { return schema.parse(JSON.parse(text)) } catch { throw new MemoryError('invalid_evaluation_review') }
}
function unique(values: string[]): void {
  if (new Set(values).size !== values.length) throw new MemoryError('duplicate_evaluation_id')
}

/** Raw-byte binding preserves the exact source and result files; it does not authenticate authors. */
export function prepareEvaluationReview(suiteText: string, reportText: string) {
  const suite = decode(suiteSchema, suiteText), report = decode(reportSchema, reportText)
  const suiteSha256 = sha256(suiteText), reportSha256 = sha256(reportText)
  if (report.suiteSha256 !== suiteSha256 || report.suite !== suite.suite) throw new MemoryError('evaluation_suite_changed')
  unique(suite.cases.map(row => row.id)); unique(report.cases.map(row => row.id))
  for (const row of report.cases) if (!suite.cases.some(f => f.id === row.id)) throw new MemoryError('unknown_evaluation_case')
  const cases = suite.cases.map(f => {
    unique(f.probes.map(p => p.id))
    const episodes = f.episodes ?? [{ id: f.id, sources: f.sources! }]
    unique(episodes.map(e => e.id))
    const originals = episodes.flatMap(e => e.sources.map((s, index) => ({
      id: `${f.id}${f.episodes ? `-${e.id}` : ''}-${index}`, projectId: f.projectId,
      episode: e.id, boundary: f.boundary ?? 'complete', ...s })))
    const result = report.cases.find(row => row.id === f.id)
    const formats = new Map<string, 'summary' | 'source_excerpts' | null>()
    if (result) {
      // Older diagnostics always captured complete episodes. A bounded fixture must have an
      // explicit matching observation; do not grade it as if omitted context had been supplied.
      if ((result.episodes?.boundary ?? 'complete') !== (f.boundary ?? 'complete')) {
        throw new MemoryError('evaluation_boundary_changed')
      }
      unique(result.records.map(record => record.id)); unique(result.probes.map(p => p.id))
      for (const p of result.probes) {
        const expected = f.probes.find(original => original.id === p.id)
        if (!expected || canonical(probe.parse(p)) !== canonical(expected)) throw new MemoryError('evaluation_probe_changed')
        unique(p.returnedIds)
        if (p.returnedIds.some(id => !result.records.some(record => record.id === id))) throw new MemoryError('unknown_recalled_memory')
        if (p.context) {
          const reviewed = reviewRecallContext({ text: p.context.text, records: result.records,
            declaredFormat: result.recallFormat, capturedSources: result.capturedSources, originals })
          const sent = reviewed.items
          formats.set(p.id, reviewed.format)
          if (p.context.bytes !== Buffer.byteLength(p.context.text, 'utf8') || p.context.bytes > p.context.maxBytes
            || canonical(sent.map(item => item.id)) !== canonical(p.returnedIds)
            || sent.some(item => !result.records.some(record => record.id === item.id && record.revision === item.revision))) {
            throw new MemoryError('invalid_recall_context')
          }
        }
      }
      if (result.probes.length !== f.probes.length) throw new MemoryError('missing_evaluation_probe')
    }
    const completed = !!result && ['learned', 'no_useful_memory'].includes(result.outcome.state)
      && result.episodes?.expected === episodes.length && result.episodes.reviewed === episodes.length
      && (result.episodes.jobs.learned ?? 0) + (result.episodes.jobs.no_useful_memory ?? 0) === episodes.length
      && result.checks.some(check => check.name === 'completed_extraction' && check.passed === true)
      && result.probes.every(p => p.status === 'ok')
    return { id: f.id, projectId: f.projectId, completed, criteria: f.expected.review,
      sources: originals,
      records: result?.records ?? [], probes: f.probes.map(p => {
        const actual = result?.probes.find(actual => actual.id === p.id)
        return { ...p, returnedIds: actual?.returnedIds ?? [],
          context: actual?.context ? { ...actual.context, format: formats.get(p.id) ?? null } : null }
      }),
      mechanicalFailures: result?.checks.filter(check => check.passed === false).map(check => check.name) ?? [] }
  })
  return { schemaVersion: 1 as const, reportSha256, suiteSha256, suite: suite.suite,
    evidence: 'development_diagnostic_only' as const,
    completed: report.status === 'completed' && cases.every(row => row.completed),
    // Hide model/arm labels in the review packet. Content itself can still reveal its origin.
    cases, review: { schemaVersion: 1 as const, reportSha256, suiteSha256, reviewer: null,
      cases: cases.filter(row => row.completed).map(row => ({ id: row.id,
        records: row.records.map(record => ({ id: record.id, revision: record.revision,
          supported: null, specific: null, useful: null, note: null })),
        probes: row.probes.map(p => ({ id: p.id, relevantIds: null, requiredIds: null, missingRequiredMemory: null,
          faithfulContext: null, note: null })) })) } }
}

type Metric = { passed: number; reviewed: number; total: number; rate: number | null }
function metric(values: Array<boolean | null>, complete: boolean): Metric {
  const reviewed = values.filter(v => v !== null).length, passed = values.filter(v => v === true).length
  return { passed, reviewed, total: values.length, rate: complete && reviewed === values.length && values.length ? passed / values.length : null }
}

/** Reviewer labels remain attributed judgements. Neither an agent nor a declared human is verified here. */
export function scoreEvaluationReview(suiteText: string, reportText: string, reviewText: string) {
  const packet = prepareEvaluationReview(suiteText, reportText), review = decode(reviewSchema, reviewText)
  if (packet.reportSha256 !== review.reportSha256 || packet.suiteSha256 !== review.suiteSha256) throw new MemoryError('evaluation_review_changed')
  unique(review.cases.map(row => row.id))
  for (const row of review.cases) if (!packet.cases.some(c => c.id === row.id && c.completed)) throw new MemoryError('case_not_reviewable')
  const memories: Array<boolean | null> = [], recalls: Array<boolean | null> = [], abstentions: Array<boolean | null> = []
  const cases = packet.cases.map(row => {
    if (!row.completed) return { id: row.id, status: 'not_reviewable' as const, probes: [] }
    const labels = review.cases.find(c => c.id === row.id)
    unique(labels?.records.map(r => r.id) ?? []); unique(labels?.probes.map(p => p.id) ?? [])
    for (const record of labels?.records ?? []) if (!row.records.some(r => r.id === record.id && r.revision === record.revision)) throw new MemoryError('review_record_changed')
    for (const p of labels?.probes ?? []) if (!row.probes.some(original => original.id === p.id)) throw new MemoryError('unknown_review_probe')
    const quality = new Map(row.records.map(record => {
      const label = labels?.records.find(r => r.id === record.id)
      const value = !review.reviewer || !label?.note || [label.supported, label.specific, label.useful].some(v => v === null)
        ? null : label.supported && label.specific && label.useful
      memories.push(value)
      return [record.id, value] as const
    }))
    const probes = row.probes.map(p => {
      const label = labels?.probes.find(v => v.id === p.id)
      for (const ids of [label?.relevantIds, label?.requiredIds]) if (ids) {
        unique(ids)
        if (ids.some(id => !quality.has(id))) throw new MemoryError('unknown_review_memory')
      }
      if (label?.relevantIds && label.requiredIds?.some(id => !label.relevantIds!.includes(id))) throw new MemoryError('invalid_required_memory')
      if (p.expected === 'abstain' && (label?.relevantIds?.length || label?.requiredIds?.length || label?.missingRequiredMemory === true)) throw new MemoryError('abstention_review_changed')
      const labelled = !!p.context && !!review.reviewer && !!label?.note && label.relevantIds !== null && label.requiredIds !== null
        && label.missingRequiredMemory !== null && label.faithfulContext !== null
        && [...(label.relevantIds ?? []), ...p.returnedIds].every(id => quality.get(id) !== null)
      const passed = !labelled ? null : !label!.faithfulContext ? false : p.expected === 'abstain' ? p.returnedIds.length === 0
        : !label!.missingRequiredMemory && p.returnedIds.some(id => label!.relevantIds!.includes(id) && quality.get(id) === true)
          && label!.requiredIds!.every(id => p.returnedIds.includes(id))
          && p.returnedIds.every(id => label!.relevantIds!.includes(id) && quality.get(id) === true)
      ;(p.expected === 'recall' ? recalls : abstentions).push(passed)
      return { id: p.id, passed }
    })
    return { id: row.id, status: 'reviewable' as const, probes }
  })
  return { schemaVersion: 1, reportSha256: packet.reportSha256, suiteSha256: packet.suiteSha256,
    evidence: packet.evidence, reviewer: review.reviewer,
    contextFormats: [...new Set(packet.cases.flatMap(row => row.probes.flatMap(p => p.context?.format ? [p.context.format] : [])))],
    coverage: { completedCases: packet.cases.filter(row => row.completed).length, expectedCases: packet.cases.length },
    memories: metric(memories, packet.completed), recall: metric(recalls, packet.completed), abstention: metric(abstentions, packet.completed),
    cases, taskBenefit: 'not_measured', nativeDelivery: 'not_measured',
    limitations: ['Ratings are attributed judgements, not authenticated independent review.',
      'Exact source and metadata checks do not establish faithful meaning or complete surrounding context.',
      'These diagnostics do not satisfy the held-out-history or paired coding-task release gates.',
      'Older reports without captured recall context can be reviewed for memory content but receive no recall-quality rate.',
      'Empty, unfinished or partially reviewed denominators never receive a passing rate.'] }
}