Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Separate memory recall presence from reviewed quality (#549)

Deecf69f796d500
diff --git a/cli/scripts/memory-extraction-eval.ts b/cli/scripts/memory-extraction-eval.ts
index 12b3535ea..42af73eff 100644
--- a/cli/scripts/memory-extraction-eval.ts
+++ b/cli/scripts/memory-extraction-eval.ts
@@ -99,6 +99,7 @@ async function run() {
limits: { maxCalls: 6, timeoutPerCallMs: 90_000, retries: 0, corpus: 'synthetic_only', tools: 'restricted_native_adapter' },
limitations: ['Development diagnostics, not held-out release evidence.',
'Mechanical checks do not establish faithful meaning or downstream task usefulness.',
+ 'Recall probe passes measure presence only; correct-memory recall requires a separately attributed semantic review.',
'Semantic review remains pending until explicitly reviewed.',
'A native init model label does not prove that the provider executed that model.',
'Native-reported cost is diagnostic metadata, not a subscription invoice.',
diff --git a/cli/scripts/memory-quality-review.ts b/cli/scripts/memory-quality-review.ts
new file mode 100644
index 000000000..39e13b313
--- /dev/null
+++ b/cli/scripts/memory-quality-review.ts
@@ -0,0 +1,42 @@
+// Offline only: --suite frozen.json --report native-report.json --output new-packet.json
+// Fill a separate copy of packet.review, then add --review labels.json to write attributed scores.
+import { open, writeFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { prepareEvaluationReview, scoreEvaluationReview } from '../src/memory/evaluationReview.js'
+import { MemoryError } from '../src/memory/types.js'
+
+async function read(path: string): Promise<string> {
+ const file = await open(path, 'r')
+ try {
+ if ((await file.stat()).size > 8_000_000) throw new MemoryError('evaluation_file_too_large')
+ return await file.readFile('utf8')
+ } finally { await file.close() }
+}
+
+try {
+ const args = process.argv.slice(2), files = new Map<string, string>()
+ for (let index = 0; index < args.length; index += 2) {
+ const key = args[index], value = args[index + 1]
+ if (!['--suite', '--report', '--review', '--output'].includes(key) || files.has(key) || !value || value.startsWith('--')) {
+ throw new MemoryError('use_suite_report_output_and_optional_review')
+ }
+ files.set(key, resolve(value))
+ }
+ if (!['--suite', '--report', '--output'].every(key => files.has(key))) throw new MemoryError('use_suite_report_output_and_optional_review')
+ const [suite, report] = await Promise.all([read(files.get('--suite')!), read(files.get('--report')!)])
+ const result = files.has('--review') ? scoreEvaluationReview(suite, report, await read(files.get('--review')!))
+ : prepareEvaluationReview(suite, report)
+ const hash = async (url: URL) => createHash('sha256').update(await read(fileURLToPath(url))).digest('hex')
+ const producer = { protocol: 'coding-memory-quality-review-v1', node: process.version,
+ runnerSha256: await hash(new URL(import.meta.url)),
+ scorerSha256: await hash(new URL('../src/memory/evaluationReview.ts', import.meta.url)) }
+ // Never overwrite an earlier result or print source conversations to the terminal.
+ await writeFile(files.get('--output')!, `${JSON.stringify({ ...result, producer }, null, 2)}\n`, { mode: 0o600, flag: 'wx' })
+ console.log(JSON.stringify({ status: 'written', kind: files.has('--review') ? 'review_scores' : 'review_packet',
+ cases: result.cases.length, nativeCalls: 0 }))
+} catch (error) {
+ console.error(JSON.stringify({ status: 'not_written', error: error instanceof MemoryError ? error.code : 'evaluation_file_unavailable' }))
+ process.exitCode = 1
+}
diff --git a/cli/src/memory/evaluation.ts b/cli/src/memory/evaluation.ts
index 06fcfd3a7..99a2e9b97 100644
--- a/cli/src/memory/evaluation.ts
+++ b/cli/src/memory/evaluation.ts
@@ -65,12 +65,15 @@ export async function evaluateExtractionCase(input: {
const probes = fixture.probes.map(probe => {
if (!completed) {
checks.push({ name: `recall:${probe.id}`, passed: null })
- return { ...probe, status: 'not_run', returnedIds: [], passed: null }
+ return { ...probe, status: 'not_run', returnedIds: [], passed: null, context: null }
}
- const result = store.recall({ query: probe.query, conditions: probe.conditions }, { profileId, projectIds: probe.projectIds, includeProfile: true })
+ const maxBytes = 3_000
+ const result = store.recall({ query: probe.query, conditions: probe.conditions, maxBytes }, { profileId, projectIds: probe.projectIds, includeProfile: true })
const passed = result.status === 'ok' && (probe.expected === 'recall' ? result.items.length > 0 : result.items.length === 0)
checks.push({ name: `recall:${probe.id}`, passed })
- return { ...probe, status: result.status, returnedIds: result.items.map(item => item.id), passed }
+ return { ...probe, status: result.status, returnedIds: result.items.map(item => item.id), passed,
+ measurement: 'presence_only' as const,
+ context: { text: result.text, bytes: Buffer.byteLength(result.text, 'utf8'), maxBytes, estimatedTokens: result.estimatedTokens } }
})
return { id: fixture.id, durationMs: Date.now() - startedAt, outcome, checks, probes, records,
episodes: { expected: episodes.length, reviewed, jobs },
diff --git a/cli/src/memory/evaluationReview.spec.ts b/cli/src/memory/evaluationReview.spec.ts
new file mode 100644
index 000000000..a26063b94
--- /dev/null
+++ b/cli/src/memory/evaluationReview.spec.ts
@@ -0,0 +1,171 @@
+import { createHash } from 'node:crypto'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, expect, it } from 'vitest'
+import { evaluateExtractionCase, type ExtractionCase } from './evaluation.js'
+import { prepareEvaluationReview, scoreEvaluationReview } from './evaluationReview.js'
+import type { MemoryDraft } from './types.js'
+
+const fixture: ExtractionCase = { id: 'example', projectId: null,
+ sources: [{ role: 'user', text: 'For debugging I prefer a small failing test first.' },
+ { role: 'user', text: 'For release notes I prefer a compatibility section.' }],
+ expected: { records: { min: 2, max: 2 }, scope: 'personal', rationale: 'null', review: ['Preserve the two conditional preferences.'] },
+ probes: [{ id: 'tests', query: 'failing test', projectIds: ['alpha'], conditions: { taskType: 'debugging' }, expected: 'recall' },
+ { id: 'unrelated', query: 'orchids', projectIds: ['alpha'], conditions: {}, expected: 'abstain' }] }
+const drafts: MemoryDraft[] = fixture.sources.map((source, index) => ({
+ kind: 'working_preference', facet: 'workflow', assertionType: 'stated_preference', scope: { profileId: 'synthetic-owner' },
+ claim: source.text, rationale: null, futureAction: source.text,
+ applicability: { taskType: index ? 'release' : 'debugging' }, exceptions: [], retrievalCues: [index ? 'release' : 'test'],
+ evidenceClass: 'user_stated', evidence: [{ sourceEventId: `example-${index}`, quote: source.text, paths: ['/claim', '/futureAction', '/applicability'] }],
+ conflictKey: index ? 'release_order' : 'debugging_order', validity: { validFrom: null, validUntil: null, recheckWhen: [] } }))
+const hash = (text: string) => createHash('sha256').update(text).digest('hex')
+let directory: string, suiteText: string, result: Awaited<ReturnType<typeof evaluateExtractionCase>>
+const report = () => JSON.stringify({ schemaVersion: 1, suite: 'review-test', suiteSha256: hash(suiteText),
+ status: 'completed', selected: { model: 'MODEL_LABEL_SHOULD_BE_HIDDEN' }, cases: [result] })
+const testing = () => result.records.find(r => r.conflictKey === 'debugging_order')!.id
+const release = () => result.records.find(r => r.conflictKey === 'release_order')!.id
+function recalled(ids: string[]) {
+ const p = result.probes[0]
+ const text = JSON.stringify({ type: 'coding_memory_context', items: ids.map(id => result.records.find(r => r.id === id)) })
+ p.returnedIds = ids
+ p.context = { text, bytes: Buffer.byteLength(text), estimatedTokens: Math.ceil(Buffer.byteLength(text) / 3), maxBytes: 3_000 }
+}
+function labels(reportText = report()) {
+ const packet = prepareEvaluationReview(suiteText, reportText)
+ return { ...packet.review, reviewer: { id: 'reviewer-a', kind: 'agent' as const, independent: false },
+ cases: packet.review.cases.map(row => ({ id: row.id,
+ records: row.records.map(r => ({ ...r, supported: true, specific: true, useful: true, note: 'The source states this conditional preference.' })),
+ probes: row.probes.map(p => ({ ...p, relevantIds: p.id === 'tests' ? [testing()] : [],
+ requiredIds: p.id === 'tests' ? [testing()] : [], missingRequiredMemory: false, faithfulContext: true,
+ note: 'Judge against the source and the frozen task conditions.' })) })) }
+}
+const score = (review = labels(), reportText = report()) => scoreEvaluationReview(suiteText, reportText, JSON.stringify(review))
+beforeEach(async () => {
+ directory = mkdtempSync(join(tmpdir(), 'memory-quality-review-'))
+ suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [fixture] })
+ result = await evaluateExtractionCase({ fixture, directory, engine: 'claude', inference: {
+ target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: drafts }),
+ } })
+})
+afterEach(() => rmSync(directory, { recursive: true, force: true }))
+
+it('prepares sources, criteria and blank labels without exposing model labels or assigning a grade', () => {
+ expect(result.checks.every(check => check.passed)).toBe(true)
+ const packet = prepareEvaluationReview(suiteText, report())
+ expect(packet.cases[0].sources.map(s => s.id)).toEqual(['example-0', 'example-1'])
+ expect(packet.cases[0].sources[0].text).toBe(fixture.sources[0].text)
+ expect(packet.review.cases[0].records.every(r => r.supported === null && r.note === null)).toBe(true)
+ expect(JSON.stringify(packet)).not.toContain('MODEL_LABEL_SHOULD_BE_HIDDEN')
+ const unreviewed = scoreEvaluationReview(suiteText, report(), JSON.stringify(packet.review))
+ expect(unreviewed.memories).toMatchObject({ total: 2, reviewed: 0, rate: null })
+ expect(unreviewed.recall.rate).toBeNull()
+})
+
+it('keeps attributed semantic review separate from native delivery and task benefit', () => {
+ const scored = score()
+ expect(scored.memories).toEqual({ total: 2, reviewed: 2, passed: 2, rate: 1 })
+ expect(scored.recall.rate).toBe(1)
+ expect(scored.abstention.rate).toBe(1)
+ expect(scored.reviewer?.kind).toBe('agent')
+ expect(scored.evidence).toBe('development_diagnostic_only')
+ expect(scored.taskBenefit).toBe('not_measured')
+ expect(scored.nativeDelivery).toBe('not_measured')
+})
+
+it('fails a nonempty recall of the wrong memory even when the older presence check passed', () => {
+ recalled([release()])
+ expect(result.probes[0].passed).toBe(true)
+ expect(score().recall).toEqual({ passed: 0, reviewed: 1, total: 1, rate: 0 })
+})
+
+it('does not hide irrelevant extras or an unmet need behind one useful recalled memory', () => {
+ recalled([testing(), release()])
+ expect(score().recall.rate).toBe(0)
+ recalled([testing()])
+ const review = labels()
+ review.cases[0].probes[0].missingRequiredMemory = true
+ expect(score(review).recall.rate).toBe(0)
+ review.cases[0].probes[0].missingRequiredMemory = false
+ review.cases[0].probes[0].faithfulContext = false
+ expect(score(review).recall.rate).toBe(0)
+})
+
+it('does not accept a relevant-looking but unsupported or unspecified memory judgement', () => {
+ const review = labels()
+ review.cases[0].records.find(r => r.id === testing())!.supported = false
+ expect(score(review).recall.rate).toBe(0)
+ expect(score(review).memories.rate).toBe(.5)
+ const partial = { ...review, cases: [{ ...review.cases[0], records: [] }] }
+ expect(score(partial).memories.rate).toBeNull()
+ expect(score(partial).recall.rate).toBeNull()
+})
+
+it('keeps an incomplete relevance judgement pending instead of requiring a particular fill order', () => {
+ const review = labels()
+ const partial = { ...review, cases: [{ ...review.cases[0], probes: review.cases[0].probes.map(p =>
+ p.id === 'tests' ? { ...p, relevantIds: null } : p) }] }
+ expect(scoreEvaluationReview(suiteText, report(), JSON.stringify(partial)).recall.rate).toBeNull()
+})
+
+it('scores completed abstention while leaving an empty memory-quality denominator unknown', async () => {
+ const empty: ExtractionCase = { ...fixture, id: 'empty', sources: [{ role: 'user', text: 'Thanks.' }],
+ expected: { ...fixture.expected, records: { min: 0, max: 0 }, scope: 'none' }, probes: [fixture.probes[1]] }
+ suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [empty] })
+ result = await evaluateExtractionCase({ fixture: empty, directory: join(directory, 'empty'), engine: 'claude', inference: {
+ target: async () => ({ state: 'ready', key: 'context' }), run: async () => JSON.stringify({ proposals: [] }),
+ } })
+ expect(score().memories).toEqual({ passed: 0, reviewed: 0, total: 0, rate: null })
+ expect(score().abstention.rate).toBe(1)
+})
+
+it('refuses changed files, record versions, duplicate labels and invented record references', () => {
+ const review = labels()
+ expect(() => score(review, `${report()}\n`)).toThrow('evaluation_review_changed')
+ expect(() => prepareEvaluationReview(`${suiteText}\n`, report())).toThrow('evaluation_suite_changed')
+ review.cases[0].records[0].revision++
+ expect(() => score(review)).toThrow('review_record_changed')
+ const duplicate = labels()
+ duplicate.cases[0].records.push(duplicate.cases[0].records[0])
+ expect(() => score(duplicate)).toThrow('duplicate_evaluation_id')
+ const invented = labels()
+ invented.cases[0].probes[0].relevantIds = ['made-up']
+ expect(() => score(invented)).toThrow('unknown_review_memory')
+})
+
+it('refuses a changed probe or a review that rewrites an abstention requirement', () => {
+ const review = labels()
+ review.cases[0].probes[1].relevantIds = [testing()]
+ expect(() => score(review)).toThrow('abstention_review_changed')
+ result.probes[0].query = 'different task'
+ expect(() => prepareEvaluationReview(suiteText, report())).toThrow('evaluation_probe_changed')
+})
+
+it('requires the actual bounded recall context before crediting correct-memory recall', () => {
+ result.probes[0].context!.bytes++
+ expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
+ result.probes[0].context!.bytes--
+ result.probes[0].context!.maxBytes = 10
+ expect(() => prepareEvaluationReview(suiteText, report())).toThrow('invalid_recall_context')
+ result.probes[0].context = null
+ expect(score().memories.rate).toBe(1)
+ expect(score().recall.rate).toBeNull()
+})
+
+it('keeps failed extraction and unfinished suites out of passing quality denominators', () => {
+ const review = labels()
+ result.outcome = { state: 'waiting_for_model' }
+ const text = report()
+ const packet = prepareEvaluationReview(suiteText, text)
+ const unavailable = scoreEvaluationReview(suiteText, text, JSON.stringify(packet.review))
+ expect(packet.review.cases).toEqual([])
+ expect(unavailable.coverage).toEqual({ completedCases: 0, expectedCases: 1 })
+ expect(unavailable.memories.rate).toBeNull()
+ expect(unavailable.abstention.rate).toBeNull()
+ expect(() => score({ ...review, reportSha256: hash(text) }, text)).toThrow('case_not_reviewable')
+ result.outcome = { state: 'learned', learned: 2 }
+ suiteText = JSON.stringify({ schemaVersion: 1, suite: 'review-test', cases: [fixture, { ...fixture, id: 'not-run' }] })
+ expect(score().coverage).toEqual({ completedCases: 1, expectedCases: 2 })
+ expect(score().memories.rate).toBeNull()
+ expect(score().recall.rate).toBeNull()
+})
diff --git a/cli/src/memory/evaluationReview.ts b/cli/src/memory/evaluationReview.ts
new file mode 100644
index 000000000..b4c0a1bd7
--- /dev/null
+++ b/cli/src/memory/evaluationReview.ts
@@ -0,0 +1,158 @@
+/** Offline semantic review of native diagnostic output. Never invokes a model or changes memory. */
+import { createHash } from 'node:crypto'
+import { z } from 'zod'
+import { canonical } from './admission.js'
+import { conditionsSchema, draftSchema, MemoryError } from './types.js'
+
+const id = z.string().min(1).max(200).regex(/^[A-Za-z0-9_.:-]+$/)
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const count = z.number().int().nonnegative().safe()
+const source = z.object({ role: z.enum(['user', 'assistant', 'tool', 'reference', 'derived']), text: z.string().min(1).max(32_000) })
+const probe = z.object({ id, query: z.string().max(4_000), projectIds: z.array(id), conditions: conditionsSchema,
+ expected: z.enum(['recall', 'abstain']) })
+const fixture = z.object({ id, projectId: id.nullable(), sources: z.array(source).min(1).optional(),
+ episodes: z.array(z.object({ id, sources: z.array(source).min(1) })).min(1).max(4).optional(),
+ expected: z.object({ review: z.array(z.string().max(2_000)) }), probes: z.array(probe) })
+ .refine(row => !!row.sources !== !!row.episodes, 'choose sources or episodes')
+const suiteSchema = z.object({ schemaVersion: z.literal(1), suite: id, cases: z.array(fixture).min(1).max(100) })
+const recordSchema = draftSchema.safeExtend({ schemaVersion: z.literal(1), id, revision: count.positive(),
+ state: z.enum(['active', 'tentative', 'needs_verification', 'superseded', 'archived']), createdAt: count, updatedAt: count })
+const contextSchema = z.object({ text: z.string().max(16_000), bytes: count, maxBytes: count.max(16_000), estimatedTokens: count })
+const packetSchema = z.object({ type: z.literal('coding_memory_context'), items: z.array(z.object({ id, revision: count.positive() })).max(6) })
+const resultSchema = z.object({ id, outcome: z.object({ state: z.string() }),
+ episodes: z.object({ expected: count, reviewed: count, jobs: z.record(z.string(), count) }).optional(),
+ checks: z.array(z.object({ name: z.string(), passed: z.boolean().nullable() })),
+ records: z.array(recordSchema).max(100),
+ probes: z.array(probe.extend({ status: z.string(), returnedIds: z.array(id).max(6), passed: z.boolean().nullable(),
+ context: contextSchema.nullable().optional() })) })
+const reportSchema = z.object({ schemaVersion: z.literal(1), suite: id, suiteSha256: hash,
+ status: z.enum(['running', 'stopped', 'completed']), cases: z.array(resultSchema).max(100) })
+const note = z.string().trim().min(1).max(2_000).nullable()
+const recordReview = z.object({ id, revision: count.positive(), supported: z.boolean().nullable(),
+ specific: z.boolean().nullable(), useful: z.boolean().nullable(), note }).strict()
+const probeReview = z.object({ id, relevantIds: z.array(id).max(100).nullable(),
+ requiredIds: z.array(id).max(100).nullable(), missingRequiredMemory: z.boolean().nullable(),
+ faithfulContext: z.boolean().nullable(), note }).strict()
+const reviewSchema = z.object({ schemaVersion: z.literal(1), reportSha256: hash, suiteSha256: hash,
+ reviewer: z.object({ id, kind: z.enum(['human', 'agent']), independent: z.boolean() }).strict().nullable(),
+ cases: z.array(z.object({ id, records: z.array(recordReview).max(100), probes: z.array(probeReview).max(100) }).strict()).max(100),
+}).strict()
+const sha256 = (text: string) => createHash('sha256').update(text).digest('hex')
+function decode<T>(schema: z.ZodType<T>, text: string): T {
+ try { return schema.parse(JSON.parse(text)) } catch { throw new MemoryError('invalid_evaluation_review') }
+}
+function unique(values: string[]): void {
+ if (new Set(values).size !== values.length) throw new MemoryError('duplicate_evaluation_id')
+}
+
+/** Raw-byte binding preserves the exact source and result files; it does not authenticate authors. */
+export function prepareEvaluationReview(suiteText: string, reportText: string) {
+ const suite = decode(suiteSchema, suiteText), report = decode(reportSchema, reportText)
+ const suiteSha256 = sha256(suiteText), reportSha256 = sha256(reportText)
+ if (report.suiteSha256 !== suiteSha256 || report.suite !== suite.suite) throw new MemoryError('evaluation_suite_changed')
+ unique(suite.cases.map(row => row.id)); unique(report.cases.map(row => row.id))
+ for (const row of report.cases) if (!suite.cases.some(f => f.id === row.id)) throw new MemoryError('unknown_evaluation_case')
+ const cases = suite.cases.map(f => {
+ unique(f.probes.map(p => p.id))
+ const episodes = f.episodes ?? [{ id: f.id, sources: f.sources! }]
+ unique(episodes.map(e => e.id))
+ const result = report.cases.find(row => row.id === f.id)
+ if (result) {
+ unique(result.records.map(record => record.id)); unique(result.probes.map(p => p.id))
+ for (const p of result.probes) {
+ const expected = f.probes.find(original => original.id === p.id)
+ if (!expected || canonical(probe.parse(p)) !== canonical(expected)) throw new MemoryError('evaluation_probe_changed')
+ unique(p.returnedIds)
+ if (p.returnedIds.some(id => !result.records.some(record => record.id === id))) throw new MemoryError('unknown_recalled_memory')
+ if (p.context) {
+ const sent = p.context.text ? decode(packetSchema, p.context.text).items : []
+ if (p.context.bytes !== Buffer.byteLength(p.context.text, 'utf8') || p.context.bytes > p.context.maxBytes
+ || canonical(sent.map(item => item.id)) !== canonical(p.returnedIds)
+ || sent.some(item => !result.records.some(record => record.id === item.id && record.revision === item.revision))) {
+ throw new MemoryError('invalid_recall_context')
+ }
+ }
+ }
+ if (result.probes.length !== f.probes.length) throw new MemoryError('missing_evaluation_probe')
+ }
+ const completed = !!result && ['learned', 'no_useful_memory'].includes(result.outcome.state)
+ && result.episodes?.expected === episodes.length && result.episodes.reviewed === episodes.length
+ && (result.episodes.jobs.learned ?? 0) + (result.episodes.jobs.no_useful_memory ?? 0) === episodes.length
+ && result.checks.some(check => check.name === 'completed_extraction' && check.passed === true)
+ && result.probes.every(p => p.status === 'ok')
+ return { id: f.id, projectId: f.projectId, completed, criteria: f.expected.review,
+ sources: episodes.flatMap(e => e.sources.map((s, index) => ({
+ id: `${f.id}${f.episodes ? `-${e.id}` : ''}-${index}`, episode: e.id, ...s }))),
+ records: result?.records ?? [], probes: f.probes.map(p => ({ ...p,
+ returnedIds: result?.probes.find(actual => actual.id === p.id)?.returnedIds ?? [],
+ context: result?.probes.find(actual => actual.id === p.id)?.context ?? null })),
+ mechanicalFailures: result?.checks.filter(check => check.passed === false).map(check => check.name) ?? [] }
+ })
+ return { schemaVersion: 1 as const, reportSha256, suiteSha256, suite: suite.suite,
+ evidence: 'development_diagnostic_only' as const,
+ completed: report.status === 'completed' && cases.every(row => row.completed),
+ // Hide model/arm labels in the review packet. Content itself can still reveal its origin.
+ cases, review: { schemaVersion: 1 as const, reportSha256, suiteSha256, reviewer: null,
+ cases: cases.filter(row => row.completed).map(row => ({ id: row.id,
+ records: row.records.map(record => ({ id: record.id, revision: record.revision,
+ supported: null, specific: null, useful: null, note: null })),
+ probes: row.probes.map(p => ({ id: p.id, relevantIds: null, requiredIds: null, missingRequiredMemory: null,
+ faithfulContext: null, note: null })) })) } }
+}
+
+type Metric = { passed: number; reviewed: number; total: number; rate: number | null }
+function metric(values: Array<boolean | null>, complete: boolean): Metric {
+ const reviewed = values.filter(v => v !== null).length, passed = values.filter(v => v === true).length
+ return { passed, reviewed, total: values.length, rate: complete && reviewed === values.length && values.length ? passed / values.length : null }
+}
+
+/** Reviewer labels remain attributed judgements. Neither an agent nor a declared human is verified here. */
+export function scoreEvaluationReview(suiteText: string, reportText: string, reviewText: string) {
+ const packet = prepareEvaluationReview(suiteText, reportText), review = decode(reviewSchema, reviewText)
+ if (packet.reportSha256 !== review.reportSha256 || packet.suiteSha256 !== review.suiteSha256) throw new MemoryError('evaluation_review_changed')
+ unique(review.cases.map(row => row.id))
+ for (const row of review.cases) if (!packet.cases.some(c => c.id === row.id && c.completed)) throw new MemoryError('case_not_reviewable')
+ const memories: Array<boolean | null> = [], recalls: Array<boolean | null> = [], abstentions: Array<boolean | null> = []
+ const cases = packet.cases.map(row => {
+ if (!row.completed) return { id: row.id, status: 'not_reviewable' as const, probes: [] }
+ const labels = review.cases.find(c => c.id === row.id)
+ unique(labels?.records.map(r => r.id) ?? []); unique(labels?.probes.map(p => p.id) ?? [])
+ for (const record of labels?.records ?? []) if (!row.records.some(r => r.id === record.id && r.revision === record.revision)) throw new MemoryError('review_record_changed')
+ for (const p of labels?.probes ?? []) if (!row.probes.some(original => original.id === p.id)) throw new MemoryError('unknown_review_probe')
+ const quality = new Map(row.records.map(record => {
+ const label = labels?.records.find(r => r.id === record.id)
+ const value = !review.reviewer || !label?.note || [label.supported, label.specific, label.useful].some(v => v === null)
+ ? null : label.supported && label.specific && label.useful
+ memories.push(value)
+ return [record.id, value] as const
+ }))
+ const probes = row.probes.map(p => {
+ const label = labels?.probes.find(v => v.id === p.id)
+ for (const ids of [label?.relevantIds, label?.requiredIds]) if (ids) {
+ unique(ids)
+ if (ids.some(id => !quality.has(id))) throw new MemoryError('unknown_review_memory')
+ }
+ if (label?.relevantIds && label.requiredIds?.some(id => !label.relevantIds!.includes(id))) throw new MemoryError('invalid_required_memory')
+ if (p.expected === 'abstain' && (label?.relevantIds?.length || label?.requiredIds?.length || label?.missingRequiredMemory === true)) throw new MemoryError('abstention_review_changed')
+ const labelled = !!p.context && !!review.reviewer && !!label?.note && label.relevantIds !== null && label.requiredIds !== null
+ && label.missingRequiredMemory !== null && label.faithfulContext !== null
+ && [...(label.relevantIds ?? []), ...p.returnedIds].every(id => quality.get(id) !== null)
+ const passed = !labelled ? null : !label!.faithfulContext ? false : p.expected === 'abstain' ? p.returnedIds.length === 0
+ : !label!.missingRequiredMemory && p.returnedIds.some(id => label!.relevantIds!.includes(id) && quality.get(id) === true)
+ && label!.requiredIds!.every(id => p.returnedIds.includes(id))
+ && p.returnedIds.every(id => label!.relevantIds!.includes(id) && quality.get(id) === true)
+ ;(p.expected === 'recall' ? recalls : abstentions).push(passed)
+ return { id: p.id, passed }
+ })
+ return { id: row.id, status: 'reviewable' as const, probes }
+ })
+ return { schemaVersion: 1, reportSha256: packet.reportSha256, suiteSha256: packet.suiteSha256,
+ evidence: packet.evidence, reviewer: review.reviewer,
+ coverage: { completedCases: packet.cases.filter(row => row.completed).length, expectedCases: packet.cases.length },
+ memories: metric(memories, packet.completed), recall: metric(recalls, packet.completed), abstention: metric(abstentions, packet.completed),
+ cases, taskBenefit: 'not_measured', nativeDelivery: 'not_measured',
+ limitations: ['Ratings are attributed judgements, not authenticated independent review.',
+ 'These diagnostics do not satisfy the held-out-history or paired coding-task release gates.',
+ 'Older reports without captured recall context can be reviewed for memory content but receive no recall-quality rate.',
+ 'Empty, unfinished or partially reviewed denominators never receive a passing rate.'] }
+}
diff --git a/docs/plans/2026-09-30-coding-memory-review-log.md b/docs/plans/2026-09-30-coding-memory-review-log.md
index 48c2d4f95..4eac12728 100644
--- a/docs/plans/2026-09-30-coding-memory-review-log.md
+++ b/docs/plans/2026-09-30-coding-memory-review-log.md
@@ -289,6 +289,11 @@ The first PR #539 CI run passed 8,001 CLI tests but failed the existing `cli.ts`
The permitted local full run completed with **8,043 passed, 6 failed and 38 skipped**. One failure was that timer assertion; all **61 checks across the switch, runtime, settings and local settings-route files** passed after its correction. The remaining five failures were the installed OpenCode TUI's missing `--auto`/`--agent` flags; the OpenCode implementation and flag tests are unchanged in this PR. The full local run is recorded as failed, not relabeled by focused rechecks. Native OpenCode compatibility and the memory model-quality gates are separate from the opt-in setting's validation.
+Later investigation for [PR #544](https://github.com/autonomous-ai/openharness/pull/544) corrected the
+missing-flag interpretation above: both flags appear in the installed binary's stderr help output.
+The unchanged test reads only stdout, which is empty. The recorded failures remain failures of that
+test; they are not evidence that the installed binary lacks those flags.
+
PR #539 merged as `7be7ae16afca5ee44da03a8bc14c1effead6eebd` after all four jobs in
[its final CI run](https://github.com/autonomous-ai/openharness/actions/runs/36857274078) passed on
head `a20f41d86aa3d09a3c7f26f9f64f6cf2683b3b4e`. No release was published.
@@ -402,3 +407,42 @@ memories and 5,000 explicitly rated receiving contexts. Across 310 requests, war
not retrieval quality or real hook latency. No personal store was seeded, native model called,
running app replaced or release published. The held-out real-history and task-benefit requirements,
real notebook faithfulness, native lifecycle coverage and fuller task/session navigation remain open.
+
+## Round 22 — distinguish recall presence from reviewed quality
+
+The next evidence question was whether the evaluation can distinguish an appropriate memory from
+any nonempty result. Its original recall probe could not: an unrelated returned record still passed
+the presence check. That mechanical result is now explicitly labelled as presence only. The learner
+diagnostic also records the exact bounded recall context so a review can examine the text an adapter
+would receive, including conditions and exceptions, rather than relying only on record IDs.
+
+An offline review tool binds source fixtures, native output and labels by exact file hashes. It
+provides source excerpts, frozen criteria, returned records and query context with blank judgements.
+It omits model/arm labels without claiming perfect blinding. Reviewers declare their identity, kind
+and independence; the tool does not authenticate those declarations. Unsupported or irrelevant
+content, missing required knowledge and unfaithful context cannot receive reviewed-recall credit.
+Incomplete annotations remain pending, and an empty or incomplete denominator never becomes 100%.
+Semantic judgements do not establish received native context, coding-task benefit or notebook quality.
+
+The latest primary-source research also argues for measuring outcomes and appropriate controls.
+The [developer-history study](https://arxiv.org/abs/2608.10319) compares personalization with generic
+and mismatched guidance, with limited personalization benefit in its setting. The
+[VibeMemBench paper](https://arxiv.org/abs/2609.23570) separates usable prior knowledge from what
+memory systems actually supply on executable coding tasks. Their findings inform our comparison
+design; neither evaluates Harness. No external dataset has been imported or counted toward our
+held-out requirement, and no actual programmer participated in this review.
+
+All **24 extraction/review tests** pass. They cover the old false-positive nonempty recall, irrelevant
+extras, missing needs, unsupported memories, unfaithful or missing context, modified files/probes,
+changed record versions, duplicate/invented IDs and incomplete extraction. A regression exposed a
+validator that rejected a partially filled annotation depending on field order; that case failed
+before the fix and now remains pending as intended. An initial missing brace in the test fixture was
+also corrected. CLI and standalone runner type checks are recorded with the final PR checks.
+
+The CLI was run against the actual saved quota-blocked diagnostic, using only its frozen synthetic
+sources. Its [review result](../research/2026-10-01-memory-quality-review-blocked.json) correctly has
+zero completed cases out of six and null memory, recall and abstention rates. A second write to the
+same path was refused and the original report hash remained unchanged. The scorer hash was checked
+against the source file. No native inference, private history scan, production memory write, app
+replacement, firmware change or release occurred. Real model quality and the full release evidence
+remain unproven; a reviewed synthetic diagnostic will not substitute for them.
diff --git a/docs/plans/2026-09-30-tim-memory.md b/docs/plans/2026-09-30-tim-memory.md
index 717abfba0..07cb32587 100644
--- a/docs/plans/2026-09-30-tim-memory.md
+++ b/docs/plans/2026-09-30-tim-memory.md
@@ -314,6 +314,41 @@ The first executable development diagnostic contains six frozen synthetic episod
Add `--batch` to select the separate [frozen multi-episode diagnostic](../research/2026-09-30-memory-batch-extraction-cases.json). Its two cases combine useful personal preferences with quoted/unbound material, and test whether an acknowledgement in a separate conversation is falsely attached to an assistant's experiment. Each captured episode retains its own session and evidence IDs. The grader requires every episode in the case to reach a successful terminal review state before scoring record counts, recall or abstention. If input limits leave part of the batch queued, quality checks stay inconclusive and semantic review is unavailable. These cases have not yet been executed against a real model; synthetic provider-output checks establish evaluation mechanics only.
+The diagnostic's original recall boolean measures **presence only**. It cannot establish that the
+right memory was returned. New reports also retain the exact recall text, byte budget and token
+estimate for review. The offline review command binds its packet and labels to the exact frozen
+suite and native report bytes, with scorer/runner hashes on its output:
+
+```bash
+cd cli
+node --import tsx scripts/memory-quality-review.ts --suite <frozen-suite.json> --report <native-report.json> --output <new-packet.json>
+node --import tsx scripts/memory-quality-review.ts --suite <frozen-suite.json> --report <native-report.json> --review <labels.json> --output <new-scores.json>
+```
+
+Make a separate copy of the packet's `review` object for labels. Attribute the reviewer as human or
+agent; judge each record's support, specificity and potential usefulness against its source and
+criteria. For each probe, identify relevant and required record IDs, any missing required memory,
+and whether the captured context faithfully preserves the claim and conditions. A relevant-looking
+but unsupported memory, an irrelevant extra, an unmet requirement or unfaithful context fails the
+reviewed recall check. Null labels remain pending. Older reports without captured context receive
+no recall-quality rate. Model/arm labels are omitted from packets, but content can reveal origin;
+reviewer identity and independence are declarations, not authenticated facts. No model is called,
+no production memory is changed, and existing output files are never overwritten.
+
+The [saved blocked-run review](../research/2026-10-01-memory-quality-review-blocked.json) has zero
+completed cases out of six and no quality percentages. Empty, unfinished and partially reviewed
+denominators do not count as success. Semantic review remains separate from actual native delivery,
+notebook faithfulness and paired coding-task outcomes; this tool does not satisfy those release gates.
+
+Recent evidence reinforces this distinction. A developer-history study reports limited and
+inconsistent gains from personalized skills compared with its controls; its replay uses a simulated
+developer and model-based grading. [Huang et al., August 2026](https://arxiv.org/abs/2608.10319).
+VibeMemBench instead grades executable repository outcomes and reports that most tested memory
+configurations did not improve on matched memory-off runs. [Fan et al., September 2026](https://arxiv.org/abs/2609.23570).
+Our inference is to include a generic-guidance control on the representative comparison subset and
+use direct injection of independently reviewed relevant knowledge to diagnose where extraction,
+retrieval or application fails. These controls are proposed comparisons, not results for Harness.
+
Run both directions, Claude → Codex and Codex → Claude, with:
1. Memory disabled, to measure ordinary task ability.
diff --git a/docs/research/2026-10-01-memory-quality-review-blocked.json b/docs/research/2026-10-01-memory-quality-review-blocked.json
new file mode 100644
index 000000000..3e74fdaa7
--- /dev/null
+++ b/docs/research/2026-10-01-memory-quality-review-blocked.json
@@ -0,0 +1,75 @@
+{
+ "schemaVersion": 1,
+ "reportSha256": "8bf50793fabd3ddad916baf8970a587dbc9abeb61106845843268b9f096d52af",
+ "suiteSha256": "135545eb05370bac04fbaf555c36255afd21fbcbd4190b0e21895e17d31273c3",
+ "evidence": "development_diagnostic_only",
+ "reviewer": null,
+ "coverage": {
+ "completedCases": 0,
+ "expectedCases": 6
+ },
+ "memories": {
+ "passed": 0,
+ "reviewed": 0,
+ "total": 0,
+ "rate": null
+ },
+ "recall": {
+ "passed": 0,
+ "reviewed": 0,
+ "total": 0,
+ "rate": null
+ },
+ "abstention": {
+ "passed": 0,
+ "reviewed": 0,
+ "total": 0,
+ "rate": null
+ },
+ "cases": [
+ {
+ "id": "conditional-debugging",
+ "status": "not_reviewable",
+ "probes": []
+ },
+ {
+ "id": "project-database-decision",
+ "status": "not_reviewable",
+ "probes": []
+ },
+ {
+ "id": "routine-tool-choice",
+ "status": "not_reviewable",
+ "probes": []
+ },
+ {
+ "id": "quoted-third-party",
+ "status": "not_reviewable",
+ "probes": []
+ },
+ {
+ "id": "unstated-rationale",
+ "status": "not_reviewable",
+ "probes": []
+ },
+ {
+ "id": "unbound-project-fact",
+ "status": "not_reviewable",
+ "probes": []
+ }
+ ],
+ "taskBenefit": "not_measured",
+ "nativeDelivery": "not_measured",
+ "limitations": [
+ "Ratings are attributed judgements, not authenticated independent review.",
+ "These diagnostics do not satisfy the held-out-history or paired coding-task release gates.",
+ "Older reports without captured recall context can be reviewed for memory content but receive no recall-quality rate.",
+ "Empty, unfinished or partially reviewed denominators never receive a passing rate."
+ ],
+ "producer": {
+ "protocol": "coding-memory-quality-review-v1",
+ "node": "v22.23.2",
+ "runnerSha256": "90252c4c052d97bdbccbd794c769a25a4e123c579afebcf6d452d58ec5b7afb9",
+ "scorerSha256": "da4c153d69c6cdf26bd8135993e576bc3f60c4042fc2c11829880b2be295bea3"
+ }
+}