Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(extract_atoms): retry and resume re-check a transcript's session source (#7869, #7870, #7868)
garrytan9952f8e64c5e
diff --git a/docs/guides/error-codes.md b/docs/guides/error-codes.mdindex 58c6d6e760..0239bfe9a0 100644--- a/docs/guides/error-codes.md+++ b/docs/guides/error-codes.md@@ -4343,6 +4343,16 @@ More: [docs/guides/write-refusals.md#timeline_rows_would_be_removed](../../docs/More: [docs/guides/data-ingestion.md#credential-redaction](../../docs/guides/data-ingestion.md#credential-redaction)+### transcript_source_mismatch++<a id="transcript_source_mismatch"></a>++| Meaning | Why | Next step | Who acts · consent | Verify | Exit | Retryable |+|---|---|---|---|---|---|---|+| An explicit atom retry was refused before it started: its transcript does not resolve to the retry's source (a captured session of another source, a session with no known source, or a transcript outside the session corpus on a source other than default). | A batch that failed before session routing may hold another source's session transcript; retrying it would write that session's atoms into the wrong source. Captured sessions resolve through their stamp, frozen record or the registered path containing their cwd, and nothing falls back to the retry's source. | Run the atom drain for the source the session belongs to (corpus_source_other), or map a held session with gbrain sweep --assign-corpus <source> (preview first) and run that source. | agent | `repeat the read that failed` | 1 | no |++Reasons: `corpus_source_other`, `corpus_source_unresolved`, `corpus_legacy_unstamped`, `corpus_stamp_missing`, `corpus_source_missing`, `corpus_source_archived`, `corpus_source_incarnation_changed`, `corpus_path_unresolvable`, `transcript_outside_corpus`.+### triage_unreliable_backoff<a id="triage_unreliable_backoff"></a>diff --git a/src/core/cycle/extract-atoms.ts b/src/core/cycle/extract-atoms.tsindex 4dd38be4e7..5608e11762 100644--- a/src/core/cycle/extract-atoms.ts+++ b/src/core/cycle/extract-atoms.ts@@ -706,16 +706,27 @@ export async function runPhaseExtractAtoms(const sourceId = opts.sourceId ?? 'default';const chat = opts._chat ?? gatewayChat;const managed = await managedAtomSession(engine, sourceId, opts._managedRetry, opts.attempt?.writeWait);- if (managed && !opts.dryRun) await resumeManagedGenerationMembers(engine, managed); // GBRA-161 PR 1b: finish what a crash after an activation left+ // v0.41.2.1: loadConfigWithEngine() so the dream.* DB-plane merge from Phase 1 reaches this phase.+ let transcriptDirs: { session_corpus_dir?: string; meeting_transcripts_dir?: string } | undefined;+ try {+ const cfg = (opts._loadConfig ? await opts._loadConfig() : await (await import('../config.ts')).loadConfigWithEngine(engine)) as unknown as+ { dream?: { synthesize?: typeof transcriptDirs } } | null;+ transcriptDirs = cfg?.dream?.synthesize;+ } catch { transcriptDirs = undefined; }+ const sourceRouting = emptySessionSourceRouting();+ if (managed && !opts.dryRun) { // GBRA-161 PR 1b: finish what a crash after an activation left+ for (const member of await resumeManagedGenerationMembers(engine, managed, { corpusDir: transcriptDirs?.session_corpus_dir ?? null })) {+ if (member.outcome !== 'held_session_source') continue;+ const held = sourceRouting.generation_members_held ??= {};+ held[member.reason!] = (held[member.reason!] ?? 0) + 1;+ }+ }const atomFiles = await derivedWriteThrough(engine, 'extract_atoms', sourceId, { managed: managed !== null, dryRun: opts.dryRun ?? false });const writeRequests: WriteReceipt[] = [];const pageAtomTitles = new Map<string, Map<string, string[]>>();// 1a. Get transcripts (test seam OR production discovery).- // v0.41.2.1: config loader switched to loadConfigWithEngine() so the- // dream.* DB-plane merge from Phase 1 reaches this phase.let transcripts: Array<{ filePath: string; content: string; contentHash: string; root?: { path: string; label: string } }> = opts._transcripts ?? [];- const sourceRouting = emptySessionSourceRouting();// Configured transcript paths are brain-global: every source walks the session// corpus and keeps only the files #6268 resolves to it (#7846); the meeting// transcripts dir stays default's.@@ -726,21 +737,13 @@ export async function runPhaseExtractAtoms() {try {const { discoverTranscripts } = await import('./transcript-discovery.ts');- const { loadConfigWithEngine } = await import('../config.ts');- const cfgRaw = opts._loadConfig- ? await opts._loadConfig()- : await loadConfigWithEngine(engine);- const cfg = (cfgRaw ?? {}) as unknown as Record<string, unknown>;- const dream = cfg.dream as- | { synthesize?: { session_corpus_dir?: string; meeting_transcripts_dir?: string } }- | undefined;- const corpusDir = dream?.synthesize?.session_corpus_dir;- const meetingDir = dream?.synthesize?.meeting_transcripts_dir;+ const corpusDir = transcriptDirs?.session_corpus_dir;+ const meetingDir = transcriptDirs?.meeting_transcripts_dir;if (corpusDir !== undefined) {const discovered = await keepSessionSourceTranscripts(engine, discoverTranscripts({corpusDir,meetingTranscriptsDir: sourceId === 'default' ? meetingDir : undefined, selfCaptureSessionIds: claudeCliSelfSessionIds(), // #5820, as synthesize- }), { corpusDir, sourceId }, sourceRouting);+ }), { corpusDir, sourceId, readOnly: opts.dryRun }, sourceRouting);transcripts = discovered.map((d) => ({filePath: d.filePath,content: d.content,diff --git a/src/core/error-registry.ts b/src/core/error-registry.tsindex bc0dc4e636..8cb5404844 100644--- a/src/core/error-registry.ts+++ b/src/core/error-registry.ts@@ -745,6 +745,7 @@ export const CODES = {timeout: { class: 'retryable', summary: "The operation did not finish within its time bound.", exit: 124 },topology_change_required: { class: 'host_only', summary: "Adding this source would replace or overlap another owner root." },transcript_secret_exposure: { class: 'unavailable', summary: "Imported conversation pages may still carry a typed credential from before the labeled-credential detector, or the last audit found some; pages are never rewritten automatically.", why: 'Re-ingest skips unchanged sessions, so pages imported before the detector keep any password typed in prose until someone reviews them.', docs: 'docs/guides/data-ingestion.md#credential-redaction', suggestion: 'Run the read-only audit to list affected slugs (never values), then review, edit or (after asking the user) remove each page, and ask the user to rotate any real credential.', fix: { argv: ['gbrain', 'transcripts', 'audit-secrets', '--json'], consent: [], actor: 'agent', why: 'Lists conversation pages that still carry a credential with hit counts per pattern; changes no page.', requires_exclusive: false, verify: { argv: ['gbrain', 'doctor', '--only', 'transcript_secret_exposure', '--json'] } } },+ transcript_source_mismatch: { class: 'caller', reasons: ['corpus_source_other', 'corpus_source_unresolved', 'corpus_legacy_unstamped', 'corpus_stamp_missing', 'corpus_source_missing', 'corpus_source_archived', 'corpus_source_incarnation_changed', 'corpus_path_unresolvable', 'transcript_outside_corpus'], summary: "An explicit atom retry was refused before it started: its transcript does not resolve to the retry's source (a captured session of another source, a session with no known source, or a transcript outside the session corpus on a source other than default).", why: "A batch that failed before session routing may hold another source's session transcript; retrying it would write that session's atoms into the wrong source. Captured sessions resolve through their stamp, frozen record or the registered path containing their cwd, and nothing falls back to the retry's source.", suggestion: 'Run the atom drain for the source the session belongs to (corpus_source_other), or map a held session with gbrain sweep --assign-corpus <source> (preview first) and run that source.' },triage_unreliable_backoff: { class: 'retryable', summary: "Dream triage did not re-judge a transcript because its last judge verdict was truncated, refused or unparseable and its backoff has not ended.", why: 'An unreliable verdict is never cached, so without a backoff the same input was paid for every cycle; it waits 24h, doubling per repeat up to 7 days, and a content or triage-model change re-judges at once.', docs: 'docs/guides/error-codes.md#triage_unreliable_backoff', suggestion: 'Leave it (it is retried after the backoff), or ask the user before `gbrain dream retriage --force`, which re-judges every transcript now and pays per file.', fix: { argv: ['gbrain', 'dream', 'retriage', '--dry-run', '--json'], consent: [], actor: 'agent', why: 'Lists the transcripts that still need a verdict without any judge call, read-only.', requires_exclusive: false } },trust_raise_refused: { class: 'consent', exit: 3, actor: 'user', summary: "A write tried to raise the trust tier of a fact, take, page or timeline entry without the owner's confirmation, so nothing was changed.", why: "Only the owner can make memory more trusted: confirming on an interactive terminal, or through a connection that holds the memory_confirm scope. The database refuses any other raise so a bug or an agent write cannot promote its own content.", docs: 'docs/guides/error-codes.md#trust_raise_refused', suggestion: 'Do not retry. Ask the user to confirm the row themselves on the brain host (gbrain trust confirm <ref>), or leave its tier as it is.' },two_consumers_on_host: { class: 'host_only', summary: "Two resident gbrain processes on this host (`serve`, `sync`, `jobs`, `autopilot` or `mcp`) have each run a full persistence consumer for longer than 30 s, so both claim writes on the same roots.", why: "One full consumer per host is a preference, not a fenced role: a `serve` always starts full, every other resident kind defers to the first live full consumer it finds in `persistence_consumers`, and two processes that start within one renewal of each other, or a CLI that took its consumer before the `serve` started, both stay full until one exits. Every observed `preparing` wedge had two consumers alive on the host. Short-lived commands (`put`, `import`, `dream`, `cli`) and rows younger than 30 s are listed, never warned.", docs: 'docs/guides/troubleshooting.md#two-consumers-on-host', suggestion: 'Let the shorter-lived of the two finish (a sync CLI run ends on its own) or restart it so it defers; with persistence.single_consumer off, this is the configured behavior.', fix: { argv: ['gbrain', 'sources', 'writer', 'status', '--json'], consent: [], actor: 'agent', why: 'Read-only: lists every consumer alive on this host (pid, kind, mode, started_at, pool) so you can see which one to let finish.', requires_exclusive: false, verify: { argv: ['gbrain', 'doctor', '--only', 'two_consumers_on_host', '--json'] } } },diff --git a/src/core/persistence/atom-maintenance.ts b/src/core/persistence/atom-maintenance.tsindex 8be26f34a8..c63c32aa28 100644--- a/src/core/persistence/atom-maintenance.ts+++ b/src/core/persistence/atom-maintenance.ts@@ -407,15 +407,18 @@ async function waitAtomReceipts(engine: BrainEngine, session: ManagedAtomSessionreturn receipts;}-export interface ResumedMember { slug: string; activation_id: string; action: 'overwrite' | 'retire'; outcome: 'waited' | 'resubmitted' | 'kept_edited' | 'unrecoverable' }+export interface ResumedMember { slug: string; activation_id: string; action: 'overwrite' | 'retire'; outcome: 'waited' | 'resubmitted' | 'kept_edited' | 'unrecoverable' | 'held_session_source'; reason?: string }/*** GBRA-161 PR 1b: finishes what a crash after an activation left pending in this source. A member request still queued* is waited on; one that ended without committing (or was never journaled) is submitted again under a fresh id bound to* the failed one, unless its page changed since the activation planned it (then the page keeps its content and the mark- * clears: a person's edit wins). Returns one entry per pending member.+ * clears: a person's edit wins). A transcript member whose session no longer resolves to this source (#7870: a generation+ * planned before session routing) is held instead: not resubmitted, its page and pending mark left as they are, reported+ * with the reason. Returns one entry per pending member.*/-export async function resumeManagedGenerationMembers(engine: BrainEngine, session: ManagedAtomSession): Promise<ResumedMember[]> {+export async function resumeManagedGenerationMembers(engine: BrainEngine, session: ManagedAtomSession,+ opts: { corpusDir: string | null }): Promise<ResumedMember[]> {const pending = await listPendingMembers(engine, [session.sourceId], atomGenerationAdapter.kind);const resumed: ResumedMember[] = [];for (const member of pending) {@@ -425,6 +428,11 @@ export async function resumeManagedGenerationMembers(engine: BrainEngine, sessioif (row.state === 'committed') { resumed.push({ slug: member.slug, activation_id: member.activation_id, action: member.action, outcome: 'waited' }); continue; }}if (row?.state === 'committed') continue;+ const origin = (row?.intent as AtomIntent | null | undefined)?.origin;+ const hold = origin?.kind === 'transcript'+ ? await (await import('../cycle/synthesize-source-routing.ts')).transcriptSourceHold(engine, origin.locator, { corpusDir: opts.corpusDir, sourceId: session.sourceId })+ : null;+ if (hold) { resumed.push({ slug: member.slug, activation_id: member.activation_id, action: member.action, outcome: 'held_session_source', reason: hold.reason }); continue; }resumed.push({ slug: member.slug, activation_id: member.activation_id, action: member.action, outcome: await resubmitMember(engine, session, member, row) });}return resumed;diff --git a/src/core/persistence/atom-retry.ts b/src/core/persistence/atom-retry.tsindex 33d42ac43d..f1fb9b3018 100644--- a/src/core/persistence/atom-retry.ts+++ b/src/core/persistence/atom-retry.ts@@ -14,6 +14,8 @@ const drainFix = (sourceId: string): Action => ({argv: ['gbrain', 'dream', '--drain', '--source', sourceId, '--json'], consent: ['paid'], actor: 'agent', requires_exclusive: false,why: `Re-reads source ${sourceId}'s current content and extracts its atom backlog under a new run; extraction calls the configured LLM.`,});+const assignFix = (sourceId: string): Action => readFix(`Lists the captured sessions held without a source and previews mapping them to ${sourceId}; writes nothing.`,+ { argv: ['gbrain', 'sweep', '--assign-corpus', sourceId] });const pageFix = (sourceId: string, slug: string): Action => readFix(`Shows which page holds ${slug} in source ${sourceId} now, with its revision, read-only.`,{ argv: ['gbrain', 'get', '--source', sourceId, '--', slug] });@@ -60,6 +62,19 @@ export async function retryManagedAtomBatch(engine: BrainEngine, sourceId: strin{ fix: drainFix(sourceId) });const retry = session.retry;const origin = retry.origin;+ if (origin.kind === 'transcript') {+ const cfg = await (await import('../config.ts')).loadConfigWithEngine(engine) as unknown as { dream?: { synthesize?: { session_corpus_dir?: string } } } | null;+ const { transcriptSourceHold } = await import('../cycle/synthesize-source-routing.ts');+ const hold = await transcriptSourceHold(engine, origin.locator, { corpusDir: cfg?.dream?.synthesize?.session_corpus_dir ?? null, sourceId });+ check();+ if (hold) throw opError('transcript_source_mismatch', `The transcript of atom batch ${requestId} does not belong to source ${sourceId}, so nothing was started.`,+ hold.owner+ ? `Transcript ${origin.locator} is a captured session of source ${hold.owner}; its atoms are extracted only by a run for that source, never retried into ${sourceId}. Run the atom drain for ${hold.owner} instead (it calls the configured LLM, so confirm the spend).`+ : hold.reason === 'transcript_outside_corpus'+ ? `Transcript ${origin.locator} is not in the session corpus, and only source default reads transcripts from outside it, so the batch is not retried into ${sourceId}. Read the batch's receipt and report it to the user.`+ : `Transcript ${origin.locator} is a captured session with no known source (${hold.reason}), so its atoms are not retried into ${sourceId}. Ask the user which source the session belongs to; gbrain sweep --assign-corpus <source> maps it, then a run for that source extracts it.`,+ { reason: hold.reason, fix: hold.owner ? drainFix(hold.owner) : hold.reason === 'transcript_outside_corpus' ? receiptFix(requestId) : assignFix(sourceId) });+ }const item = origin.kind === 'page'? { kind: 'page' as const, slug: origin.locator, content: (await engine.readPageSnapshot(origin.locator, { sourceId }))?.page.compiled_truth ?? '', contentHash: origin.contentHash }: { kind: 'transcript' as const, filePath: origin.locator, content: readFileSync(origin.locator, 'utf8'), contentHash: origin.contentHash };diff --git a/test/atom-retry-session-source-7869.test.ts b/test/atom-retry-session-source-7869.test.tsnew file mode 100644index 0000000000..9b001d03c2--- /dev/null+++ b/test/atom-retry-session-source-7869.test.ts@@ -0,0 +1,96 @@+/**+ * #7869: an explicit managed atom retry (`gbrain jobs submit extract-atoms-drain+ * --params '{"retryRequestId": …}'`) re-checks a transcript item's session+ * source before it starts anything. A batch that failed before #7851 may hold+ * another source's session transcript, extracted into the run's source; the+ * retry passed it to the phase through the `_transcripts` seam, which skips+ * `keepSessionSourceTranscripts`, so the rerun wrote that session's atoms into+ * the request's source. Now a transcript that does not resolve to the retry's+ * source (#6268: stamp, frozen record, tier-4 cwd) is refused with+ * `transcript_source_mismatch`: no model call, no write. A session of the+ * retry's own source still retries.+ *+ * Managed brain, PGLite and, with DATABASE_URL, Postgres.+ */+import { describe, expect, test } from 'bun:test';+import { mkdirSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';+import type { BrainEngine } from '../src/core/engine.ts';+import { __setChatTransportForTests, type ChatResult } from '../src/core/ai/gateway.ts';+import { runPhaseExtractAtoms } from '../src/core/cycle/extract-atoms.ts';+import { CORPUS_UNRESOLVED_STAMP, ensureCorpusSpoolDir, sessionCorpusFileName } from '../src/core/context/corpus-segments.ts';+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';+import { sha256 } from '../src/core/persistence/digest.ts';+import { retryManagedAtomBatch } from '../src/core/persistence/atom-retry.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';++const VALID = JSON.stringify([{ title: 'Measured progress', atom_type: 'insight', body: 'Measure progress against clear exit criteria.' }]);+const TEXT = 'A careful session transcript about measured progress. '.repeat(40);++function reply(text: string): ChatResult {+ return { text, blocks: [{ type: 'text', text }], stopReason: 'end',+ usage: { input_tokens: 10, output_tokens: 10, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'anthropic:claude-haiku-4-5', providerId: 'anthropic' };+}++/** A pre-#7851 run: the transcript reaches the phase through the seam, unrouted, and the model reply is malformed. */+async function failedBatch(engine: BrainEngine, filePath: string): Promise<string> {+ writeFileSync(filePath, TEXT);+ const result = await runPhaseExtractAtoms(engine, { sourceId: 'default', _pages: [],+ _transcripts: [{ filePath, content: TEXT, contentHash: sha256(TEXT) }], _chat: async () => reply('not valid output') });+ expect(result.details.malformed_outputs).toBe(1);+ await disposePersistenceConsumer(engine);+ const [row] = await engine.executeRaw<{ request_id: string }>(+ `SELECT completed_keys->0->>'requestId' AS request_id FROM op_checkpoints WHERE op='managed-atoms' AND completed_keys->0->>'failure' IS NOT NULL+ AND completed_keys::text LIKE '%' || $1 || '%' ORDER BY updated_at DESC LIMIT 1`, [filePath.split('/').pop()]);+ expect(row?.request_id).toBeTruthy();+ return row!.request_id;+}++const atoms = (engine: BrainEngine) => engine.executeRaw<{ source_id: string; slug: string }>(+ "SELECT source_id, slug FROM pages WHERE type='atom' AND deleted_at IS NULL ORDER BY slug");++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;+ describe(`#7869 explicit atom retry honors the session source (${backend})`, () => {+ test('a failed batch of another source\'s session, or of an unresolved one, is refused; nothing starts', async () => {+ await managedBrain(async ({ engine, root }) => {+ const corpus = join(dirname(root), 'corpus');+ mkdirSync(corpus);+ await engine.setConfig('dream.synthesize.session_corpus_dir', corpus);+ const spool = ensureCorpusSpoolDir(corpus);+ let calls = 0;+ __setChatTransportForTests(async () => { calls++; return reply(VALID); });+ try {+ for (const [file, reason] of [+ [join(spool, sessionCorpusFileName('sess-other', 'source-b')), 'corpus_source_other'],+ [join(spool, sessionCorpusFileName('sess-unresolved', CORPUS_UNRESOLVED_STAMP)), 'corpus_source_unresolved'],+ ] as const) {+ const requestId = await failedBatch(engine, file);+ const error = await retryManagedAtomBatch(engine, 'default', requestId, `job:7869-${reason}`).catch(e => e);+ expect(error).toMatchObject({ code: 'transcript_source_mismatch', reason });+ expect(String(error.message)).toContain('nothing was started');+ expect(calls).toBe(0);+ expect(await atoms(engine)).toEqual([]);+ }+ } finally { __setChatTransportForTests(null); }+ }, { databaseUrl, setup: ({ engine }) => engine.executeRaw("INSERT INTO sources(id,name) VALUES('source-b','source-b')").then(() => undefined) });+ }, 180_000);++ test('a failed batch of the retry\'s own session source still reruns and extracts', async () => {+ await managedBrain(async ({ engine, root }) => {+ const corpus = join(dirname(root), 'corpus');+ mkdirSync(corpus);+ await engine.setConfig('dream.synthesize.session_corpus_dir', corpus);+ const requestId = await failedBatch(engine, join(ensureCorpusSpoolDir(corpus), sessionCorpusFileName('sess-own', 'default')));+ let calls = 0;+ __setChatTransportForTests(async () => { calls++; return reply(VALID); });+ try {+ expect(await retryManagedAtomBatch(engine, 'default', requestId, 'job:7869-own')).toMatchObject({ model_rerun: true });+ } finally { __setChatTransportForTests(null); }+ expect(calls).toBe(1);+ expect((await atoms(engine)).map(row => row.source_id)).toEqual(['default']);+ }, { databaseUrl });+ }, 180_000);+ });+}diff --git a/test/corpus-source-deferred-7868.test.ts b/test/corpus-source-deferred-7868.test.tsnew file mode 100644index 0000000000..ca0f2b192d--- /dev/null+++ b/test/corpus-source-deferred-7868.test.ts@@ -0,0 +1,199 @@+/**+ * #7868: an unresolved session's deferred resolution accepts only tier-4+ * evidence, a registered source whose local_path contains the session's+ * recorded cwd (active over archived). Before the fix it ran the whole+ * resolution chain, whose last tiers are `sources.default`, the sole+ * non-default convenience and finally `default`, and froze the answer: a+ * session captured anywhere else on a multi-source brain was pinned to the+ * fallback for good. A session with no tier-4 evidence now stays held as+ * `corpus_source_unresolved` (nothing frozen) until `gbrain sweep+ * --assign-corpus <source>` maps it. A single-source brain still resolves its+ * only source, without freezing, so a second source added later holds the+ * session instead of inheriting a stale pin.+ *+ * #7850: `resolveCorpusFileSource(..., { readOnly: true })` (every dry run)+ * freezes no record and binds no incarnation; an extract_atoms dry run passes it.+ *+ * Repo-less brain directories (no Git checkout); PGLite and, with+ * DATABASE_URL, Postgres.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { BrainEngine } from '../src/core/engine.ts';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { runPhaseExtractAtoms } from '../src/core/cycle/extract-atoms.ts';+import { CORPUS_UNRESOLVED_STAMP, corpusSpoolDir, ensureCorpusSpoolDir, sessionCorpusFileName } from '../src/core/context/corpus-segments.ts';+import { assignCorpusSessions, freezeSessionSource, listCorpusSourceHolds, openSessionSource, readSessionSource, resolveCorpusFileSource } from '../src/core/context/corpus-source.ts';+import type { GBrainConfig } from '../src/core/config.ts';+import type { ChatResult } from '../src/core/ai/gateway.ts';+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';+import { withEnv } from './helpers/with-env.ts';+import { testBackends } from './helpers/test-backends.ts';++const backends = testBackends();+const engines: BrainEngine[] = [];+let dataDir: string;+let closePostgres: (() => Promise<void>) | undefined;+beforeAll(async () => {+ dataDir = mkdtempSync(join(tmpdir(), 'gbrain-deferred-source-db-'));+ if (backends.includes('pglite')) {+ const engine = new PGLiteEngine();+ await engine.connect({ database_path: dataDir });+ await engine.initSchema();+ engines.push(engine);+ }+ if (backends.includes('postgres')) {+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);+ engines.push(pg.engine);+ closePostgres = pg.close;+ }+}, 120_000);+afterAll(async () => {+ for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }+ await closePostgres?.();+ rmSync(dataDir, { recursive: true, force: true });+});++interface Brain { engine: BrainEngine; dir: string; corpus: string; a: string; b: string }++async function brain(run: (b: Brain) => Promise<void>): Promise<void> {+ for (const engine of engines) {+ const dir = mkdtempSync(join(tmpdir(), 'gbrain-deferred-source-'));+ const corpus = join(dir, 'corpus');+ mkdirSync(corpus);+ const tag = randomUUID().slice(0, 8);+ try {+ await withEnv({ GBRAIN_HOME: join(dir, 'home'), GBRAIN_SOURCE: undefined }, async () => {+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ await run({ engine, dir, corpus, a: `source-a-${tag}`, b: `source-b-${tag}` });+ });+ } finally {+ await engine.executeRaw("DELETE FROM sources WHERE id <> 'default'");+ await engine.executeRaw("DELETE FROM config WHERE key = 'sources.default'");+ rmSync(dir, { recursive: true, force: true });+ }+ }+}++const TRANSCRIPT = `[user]\nwe keep the quarterly plan in one place.\n\n${'[assistant]\nNoted, we will measure it for two weeks.\n\n'.repeat(60)}`;++/** A captured session no client-side tier resolved: its record keeps the cwd, its file is stamped unresolved. */+async function unresolvedSession(corpus: string, sessionId: string, cwd: string): Promise<string> {+ mkdirSync(cwd, { recursive: true });+ expect((await openSessionSource(corpus, sessionId, { cwd, harness: 'test' })).stamp).toBe(CORPUS_UNRESOLVED_STAMP);+ const name = sessionCorpusFileName(sessionId, CORPUS_UNRESOLVED_STAMP);+ writeFileSync(join(ensureCorpusSpoolDir(corpus), name), TRANSCRIPT);+ return name;+}++const addSource = (engine: BrainEngine, id: string, localPath: string | null = null, archived = false) =>+ engine.executeRaw('INSERT INTO sources(id,name,local_path,archived) VALUES($1,$1,$2,$3)', [id, localPath, archived]);++describe('#7868 deferred session resolution accepts only tier-4 evidence', () => {+ test('a cwd no registered source contains stays held: never the configured default, never default; --assign-corpus releases it', async () => {+ await brain(async ({ engine, dir, corpus, a, b }) => {+ await addSource(engine, a);+ await addSource(engine, b);+ await engine.setConfig('sources.default', b);+ const configured = await unresolvedSession(corpus, 'sess-configured', join(dir, 'ws-1'));+ expect(await resolveCorpusFileSource(engine, corpus, configured, 'spool')).toEqual({ ok: false, reason: 'corpus_source_unresolved' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-configured')).toMatchObject({ source_id: null, tier: null });++ await engine.executeRaw("DELETE FROM config WHERE key = 'sources.default'");+ const seeded = await unresolvedSession(corpus, 'sess-seeded', join(dir, 'ws-2'));+ expect(await resolveCorpusFileSource(engine, corpus, seeded, 'spool')).toEqual({ ok: false, reason: 'corpus_source_unresolved' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-seeded')).toMatchObject({ source_id: null, tier: null });+ expect(listCorpusSourceHolds(corpus, false)).toMatchObject({ unresolved: 2, sessions: ['sess-configured', 'sess-seeded'] });++ expect(assignCorpusSessions(corpus, ['sess-configured', 'sess-seeded'], a)).toEqual(['sess-configured', 'sess-seeded']);+ expect(await resolveCorpusFileSource(engine, corpus, configured, 'spool')).toMatchObject({ ok: true, sourceId: a, via: 'session_record' });+ expect(await resolveCorpusFileSource(engine, corpus, seeded, 'spool')).toMatchObject({ ok: true, sourceId: a, via: 'session_record' });+ });+ }, 120_000);++ test('the sole non-default convenience is not evidence either', async () => {+ await brain(async ({ engine, dir, corpus, a }) => {+ await addSource(engine, a, join(dir, 'brain-a'));+ const name = await unresolvedSession(corpus, 'sess-elsewhere', join(dir, 'ws'));+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toEqual({ ok: false, reason: 'corpus_source_unresolved' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-elsewhere')?.source_id).toBeNull();+ });+ }, 120_000);++ test('tier 4: the deepest active registered path containing the cwd resolves and freezes; an archived deeper path loses', async () => {+ await brain(async ({ engine, dir, corpus, a, b }) => {+ await addSource(engine, a, join(dir, 'ws'));+ await addSource(engine, b, join(dir, 'ws', 'deep'), true);+ const name = await unresolvedSession(corpus, 'sess-inside', join(dir, 'ws', 'deep', 'task'));+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toMatchObject({ ok: true, sourceId: a, via: 'session_record' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-inside')).toMatchObject({ source_id: a, tier: 'engine' });+ });+ }, 120_000);++ test('a cwd only an archived source contains stays held, unfrozen', async () => {+ await brain(async ({ engine, dir, corpus, a, b }) => {+ await addSource(engine, a);+ await addSource(engine, b, join(dir, 'ws'), true);+ const name = await unresolvedSession(corpus, 'sess-archived', join(dir, 'ws', 'task'));+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toEqual({ ok: false, reason: 'corpus_source_unresolved' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-archived')?.source_id).toBeNull();+ });+ }, 120_000);++ test('a single-source brain resolves its only source without freezing; a second source then holds the session', async () => {+ await brain(async ({ engine, dir, corpus, a }) => {+ const name = await unresolvedSession(corpus, 'sess-solo', join(dir, 'ws'));+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toMatchObject({ ok: true, sourceId: 'default', via: 'sole_source' });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-solo')).toMatchObject({ source_id: null, source_incarnation: null });+ expect(listCorpusSourceHolds(corpus, true).unresolved).toBe(0);++ await addSource(engine, a);+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toEqual({ ok: false, reason: 'corpus_source_unresolved' });+ expect(listCorpusSourceHolds(corpus, false).sessions).toEqual(['sess-solo']);+ });+ }, 120_000);+});++describe('#7850 the read-only resolver writes no session record', () => {+ test('readOnly resolves tier-4 evidence without freezing, and binds no incarnation on a frozen record', async () => {+ await brain(async ({ engine, dir, corpus, a }) => {+ await addSource(engine, a, join(dir, 'ws'));+ const name = await unresolvedSession(corpus, 'sess-preview', join(dir, 'ws', 'task'));+ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool', { readOnly: true })).toMatchObject({ ok: true, sourceId: a });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-preview')).toMatchObject({ source_id: null, tier: null });++ expect(freezeSessionSource(corpus, 'sess-bound', a, 'operator')?.source_incarnation).toBeNull();+ const bound = sessionCorpusFileName('sess-bound', a);+ writeFileSync(join(ensureCorpusSpoolDir(corpus), bound), '[user]\nthe weekly plan\n');+ expect(await resolveCorpusFileSource(engine, corpus, bound, 'spool', { readOnly: true })).toMatchObject({ ok: true, sourceId: a });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-bound')?.source_incarnation).toBeNull();++ expect(await resolveCorpusFileSource(engine, corpus, name, 'spool')).toMatchObject({ ok: true, sourceId: a });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-preview')).toMatchObject({ source_id: a, tier: 'engine' });+ expect(await resolveCorpusFileSource(engine, corpus, bound, 'spool')).toMatchObject({ ok: true, sourceId: a });+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-bound')?.source_incarnation).not.toBeNull();+ });+ }, 120_000);++ test('an extract_atoms dry run lists the session under its source and leaves its record unresolved', async () => {+ await brain(async ({ engine, dir, corpus, a }) => {+ await addSource(engine, a, join(dir, 'ws'));+ const brainDir = join(dir, 'brain-a');+ mkdirSync(brainDir);+ await unresolvedSession(corpus, 'sess-dry', join(dir, 'ws', 'task'));+ const result = await runPhaseExtractAtoms(engine, {+ sourceId: a, brainDir, dryRun: true, _pages: [],+ _loadConfig: () => ({ dream: { synthesize: { session_corpus_dir: corpus } } }) as unknown as GBrainConfig,+ _chat: async (): Promise<ChatResult> => ({ text: '[]', blocks: [{ type: 'text', text: '[]' }], stopReason: 'end',+ usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'anthropic:claude-haiku-4-5', providerId: 'anthropic' }),+ });+ expect(result.details.transcripts_total).toBe(1);+ expect(result.details.source_routing).toBeUndefined();+ expect(readSessionSource(corpusSpoolDir(corpus), 'sess-dry')).toMatchObject({ source_id: null, tier: null });+ });+ }, 120_000);+});diff --git a/test/e2e/atom-retry-session-source-7869.test.ts b/test/e2e/atom-retry-session-source-7869.test.tsnew file mode 100644index 0000000000..b7fa7e2dd2--- /dev/null+++ b/test/e2e/atom-retry-session-source-7869.test.ts@@ -0,0 +1,3 @@+import { registerPostgresTests } from '../helpers/test-backends.ts';++await registerPostgresTests(() => import('../atom-retry-session-source-7869.test.ts'));diff --git a/test/e2e/corpus-source-deferred-7868.test.ts b/test/e2e/corpus-source-deferred-7868.test.tsnew file mode 100644index 0000000000..e8d0e2b8a4--- /dev/null+++ b/test/e2e/corpus-source-deferred-7868.test.ts@@ -0,0 +1,3 @@+import { registerPostgresTests } from '../helpers/test-backends.ts';++await registerPostgresTests(() => import('../corpus-source-deferred-7868.test.ts'));diff --git a/test/e2e/extract-atoms-resume-session-source-7870.test.ts b/test/e2e/extract-atoms-resume-session-source-7870.test.tsnew file mode 100644index 0000000000..f288c51bc4--- /dev/null+++ b/test/e2e/extract-atoms-resume-session-source-7870.test.ts@@ -0,0 +1,3 @@+import { registerPostgresTests } from '../helpers/test-backends.ts';++await registerPostgresTests(() => import('../extract-atoms-resume-session-source-7870.test.ts'));diff --git a/test/extract-atoms-resume-session-source-7870.test.ts b/test/extract-atoms-resume-session-source-7870.test.tsnew file mode 100644index 0000000000..eefaa140e1--- /dev/null+++ b/test/extract-atoms-resume-session-source-7870.test.ts@@ -0,0 +1,120 @@+/**+ * #7870: the start of every managed extract_atoms run resumes generation+ * members a crash left pending (`resumeManagedGenerationMembers`). A+ * generation planned before #7851 may have extracted another source's session+ * transcript into the run's source; resuming its members resubmitted them+ * with no routing check, so the re-run overwrote or retired atoms on that+ * session's behalf. Now a transcript-derived member whose session no longer+ * resolves to the run's source (#6268) is held: not resubmitted, its atom page+ * left exactly as it is (moving misplaced atoms is #7845), and counted in the+ * phase's `details.source_routing.generation_members_held`. A member of the+ * run's own session source still resumes.+ *+ * Managed brain, PGLite and, with DATABASE_URL, Postgres.+ */+import { describe, expect, test } from 'bun:test';+import { mkdirSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';+import type { ChatResult } from '../src/core/ai/gateway.ts';+import type { BrainEngine } from '../src/core/engine.ts';+import type { GBrainConfig } from '../src/core/config.ts';+import { runPhaseExtractAtoms } from '../src/core/cycle/extract-atoms.ts';+import { ensureCorpusSpoolDir, sessionCorpusFileName } from '../src/core/context/corpus-segments.ts';+import { disposePersistenceConsumer, waitForWrites } from '../src/core/persistence/service.ts';+import type { WriteRequest } from '../src/core/persistence/model.ts';+import { sha256 } from '../src/core/persistence/digest.ts';+import { installFaultHook } from '../src/core/persistence/fault-points.ts';+import { OperationError } from '../src/core/ops/contract.ts';+import { listPendingMembers } from '../src/core/generations/managed.ts';+import { managedBrain, settleManagedGitEffects } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';++const body = (label: string) => (label + ': the team ships small changes, measures each launch and reviews outcomes weekly. ').repeat(12);++function stub(titles: string[]) {+ return async (): Promise<ChatResult> => {+ const text = JSON.stringify(titles.map(title => ({ title, body: title + '.', atom_type: 'insight' })));+ return { text, blocks: [{ type: 'text', text }], stopReason: 'end',+ usage: { input_tokens: 10, output_tokens: 10, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'anthropic:claude-haiku-4-5', providerId: 'anthropic' };+ };+}++/** A pre-#7851 run: the session transcript reaches the phase through the seam, unrouted. */+async function extractUnrouted(engine: BrainEngine, filePath: string, label: string, titles: string[]): Promise<void> {+ const content = body(label);+ writeFileSync(filePath, content);+ await runPhaseExtractAtoms(engine, { sourceId: 'default', _pages: [], _chat: stub(titles),+ _transcripts: [{ filePath, content, contentHash: sha256(content) }] });+ await disposePersistenceConsumer(engine);+}++async function drain(engine: BrainEngine): Promise<void> {+ const pending = await engine.executeRaw<WriteRequest>("SELECT * FROM persistence_requests WHERE source_id='default' AND state IN ('queued','running','recovering')");+ await waitForWrites(engine, pending, { engine: engine.kind } as GBrainConfig, 60_000);+ await disposePersistenceConsumer(engine);+}++const atoms = (engine: BrainEngine) => engine.executeRaw<{ slug: string; live: boolean; retired_by: string | null }>(+ "SELECT slug, deleted_at IS NULL AS live, frontmatter->>'retired_by' AS retired_by FROM pages WHERE source_id='default' AND type='atom' ORDER BY slug");++/** Two unrouted extractions of one session file, the second crashing after its activation: its retire members stay pending. */+async function crashedGeneration(engine: BrainEngine, filePath: string): Promise<string[]> {+ await extractUnrouted(engine, filePath, 'first', ['Ship small changes', 'Review outcomes weekly', 'Measure each launch', 'Pair on reviews']);+ await settleManagedGitEffects(engine);+ const retiring = (await atoms(engine)).filter(row => /measure-each-launch|pair-on-reviews/.test(row.slug)).map(row => row.slug).sort();+ expect(retiring).toHaveLength(2);+ let crashes = 0;+ installFaultHook(async point => { if (point === 'generation:member') { crashes++; throw new OperationError('revision_conflict', 'Simulated crash: the member request never landed.', 'Run the atom drain again.'); } });+ try { await extractUnrouted(engine, filePath, 'second', ['Ship small changes', 'Review outcomes weekly', 'Track error budgets']).catch(() => undefined); await drain(engine); }+ finally { installFaultHook(undefined); }+ expect(crashes).toBeGreaterThan(0);+ expect((await listPendingMembers(engine, ['default'], 'atom')).filter(row => row.action === 'retire').map(row => row.slug).sort()).toEqual(retiring);+ return retiring;+}++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;+ describe(`#7870 resumed generation members honor the session source (${backend})`, () => {+ test('a pending member of another source\'s session is held, never resubmitted; its atom stays live', async () => {+ await managedBrain(async ({ engine, root }) => {+ const corpus = join(dirname(root), 'corpus');+ mkdirSync(corpus);+ await engine.setConfig('dream.synthesize.session_corpus_dir', corpus);+ const loadConfig = () => ({ dream: { synthesize: { session_corpus_dir: corpus } } }) as unknown as GBrainConfig;+ const retiring = await crashedGeneration(engine, join(ensureCorpusSpoolDir(corpus), sessionCorpusFileName('sess-other', 'source-b')));+ const pending = (await listPendingMembers(engine, ['default'], 'atom')).map(row => [row.slug, row.action]);+ expect(pending.length).toBeGreaterThan(retiring.length);++ const results = [];+ for (let run = 0; run < 2; run++) {+ results.push(await runPhaseExtractAtoms(engine, { sourceId: 'default', _pages: [], _transcripts: [], _loadConfig: loadConfig, _chat: stub([]) }));+ await disposePersistenceConsumer(engine);+ }+ const after = await atoms(engine);+ for (const slug of retiring) expect(after.find(row => row.slug === slug)).toMatchObject({ live: true, retired_by: null });+ expect(await engine.executeRaw("SELECT slug FROM persistence_requests WHERE source_id='default' AND intent->>'kind'='managed_atom_delete' AND state='committed'")).toEqual([]);+ expect((await listPendingMembers(engine, ['default'], 'atom')).map(row => [row.slug, row.action])).toEqual(pending);+ for (const result of results) {+ expect(result.details.source_routing).toMatchObject({ source_id: 'default', generation_members_held: { corpus_source_other: pending.length } });+ expect(result.summary).toContain(`${pending.length} pending generation member(s) held`);+ }+ }, { databaseUrl, setup: ({ engine }) => engine.executeRaw("INSERT INTO sources(id,name) VALUES('source-b','source-b')").then(() => undefined) });+ }, 240_000);++ test('a pending member of the run\'s own session source still resumes', async () => {+ await managedBrain(async ({ engine, root }) => {+ const corpus = join(dirname(root), 'corpus');+ mkdirSync(corpus);+ await engine.setConfig('dream.synthesize.session_corpus_dir', corpus);+ const loadConfig = () => ({ dream: { synthesize: { session_corpus_dir: corpus } } }) as unknown as GBrainConfig;+ const retiring = await crashedGeneration(engine, join(ensureCorpusSpoolDir(corpus), sessionCorpusFileName('sess-own', 'default')));+ const result = await runPhaseExtractAtoms(engine, { sourceId: 'default', _pages: [], _transcripts: [], _loadConfig: loadConfig, _chat: stub([]) });+ await disposePersistenceConsumer(engine);+ expect(result.details.source_routing).toBeUndefined();+ const after = await atoms(engine);+ for (const slug of retiring) expect(after.find(row => row.slug === slug)).toMatchObject({ live: false, retired_by: 'managed-reextract' });+ expect(await listPendingMembers(engine, ['default'], 'atom')).toEqual([]);+ }, { databaseUrl });+ }, 240_000);+ });+}