Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

feat(persistence): a conversion apply reports progress, a partial report on SIGTERM, and GBRAIN_MEMORY_TRACE (#7644)

garrytan416a5c3d72e3
diff --git a/scripts/bench/sync-pass-memory.ts b/scripts/bench/sync-pass-memory.ts
new file mode 100644
index 0000000000..0974150362
--- /dev/null
+++ b/scripts/bench/sync-pass-memory.ts
@@ -0,0 +1,250 @@
+#!/usr/bin/env bun
+/**
+ * Probe: memory of the managed sync pass on a large source (#7644, #7596).
+ *
+ * bun scripts/bench/sync-pass-memory.ts seed --url <admin pg url> --dir <dir> [--pages 20000] [--ahead 5000] [--commits 2000] [--dbonly 2000]
+ * bun scripts/bench/sync-pass-memory.ts run --url <admin pg url> --dir <dir> [--mode convert|sync] [--extract] [--full] [--strict]
+ * bun scripts/bench/sync-pass-memory.ts cli ... [--mode convert|sync] [--embed] [--extract] [--full] [--kill-after <s>] [--db-url <pooler url>]
+ * bun scripts/bench/sync-pass-memory.ts embed-seed ... (embeds the template's pages through the stub, as a brain that already has vectors)
+ * (--tag <name> names the template and run databases, so several seeds can live side by side)
+ * cli runs the real gbrain CLI as a child (GBRAIN_MEMORY_TRACE on, embeddings through a loopback stub with --embed) and
+ * samples its RSS from /proc once a second.
+ *
+ * seed builds a managed Postgres brain whose default source holds --pages committed files written over --commits commits
+ * (with renames and rewrites, then git gc), first-syncs it, makes --ahead of its pages database_ahead with no import
+ * record (the row changes only type, frontmatter.legacy_type and frontmatter.subtype), adds --dbonly database-only pages,
+ * and keeps the result as a template database. run clones the template, resets the checkout to the seeded commit and runs
+ * the conversion apply (convert) or a plain full sync (sync), sampling RSS and heap once a second.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, rmSync, writeFileSync, appendFileSync, renameSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import postgres from '#postgres';
+import { PostgresEngine } from '../../src/core/postgres-engine.ts';
+import { DEFAULTS, pageContent, pagePath, planSource } from './production-soak/fixture.ts';
+import { startEmbeddingStub } from './managed-sync-catchup-lib.ts';
+import { readFileSync } from 'node:fs';
+
+const args = process.argv.slice(2);
+const flag = (name: string, fallback?: string) => { const i = args.indexOf('--' + name); return i >= 0 ? args[i + 1]! : fallback; };
+const command = args[0];
+const adminUrl = flag('url', 'postgresql://postgres:[email protected]:5434/postgres')!;
+const dir = flag('dir')!;
+const PAGES = Number(flag('pages', '20000')), AHEAD = Number(flag('ahead', '5000')), COMMITS = Number(flag('commits', '200')), DBONLY = Number(flag('dbonly', '0'));
+const KB = Number(flag('kb', '1')), LEGACY = Number(flag('legacy', '0')), FIXTURE = args.includes('--fixture'), EXTRACT = args.includes('--extract');
+// --fixture: the production-soak fixture's pages (links, facts fences, timelines, private pages, long meetings and chats), its default source.
+const soak = { ...DEFAULTS, pages: Math.ceil(PAGES / 0.52) + 1, hugeMb: 1, oversized: 0 };
+const soakPlan = FIXTURE ? planSource(soak, 0) : null;
+if (soakPlan && soakPlan.lives < PAGES) throw new Error(`the fixture plan has ${soakPlan.lives} pages, fewer than --pages`);
+const tag = (flag('tag', 'probe') ?? 'probe').replace(/[^a-z0-9_]/gi, '_').toLowerCase();
+const SEED_DB = `gbrain_${tag}_seed`, RUN_DB = `gbrain_${tag}_run`;
+const home = join(dir, 'home'), root = join(home, 'content');
+process.env.GBRAIN_HOME = home;
+delete process.env.DATABASE_URL; delete process.env.GBRAIN_DATABASE_URL;
+
+const git = (...a: string[]) => execFileSync('git', ['-C', root, '-c', 'commit.gpgsign=false', ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 1 << 28 }).trim();
+const dbUrl = (name: string) => { const u = new URL(adminUrl); u.pathname = '/' + name; return u.toString(); };
+const WORDS = 'alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega'.split(' ');
+const body = (i: number, v = 0) => Array.from({ length: Math.round((30 + (i % 40)) * KB * 1.5) }, (_, k) => (k % 12 === 11 ? '\n\n' : '') + WORDS[(i * 7 + k * 13 + v) % WORDS.length]).join(' ');
+const slugOf = (i: number) => 'notes/d' + String(i % 50).padStart(2, '0') + '/p' + String(i).padStart(6, '0');
+const file = (i: number, v = 0) => '---\ntype: note\ntitle: Page ' + i + '\n---\n\n# Page ' + i + '\n\n' + body(i, v) + '\n\n' + body(i + 1, v) + '\n';
+const pathOf = (i: number) => soakPlan ? pagePath(soak, soakPlan, i, 0) : slugOf(i) + '.md';
+const contentOf = (i: number, v = 0) => soakPlan ? pageContent(soak, soakPlan, i, v) : file(i, v);
+const write = (path: string, text: string) => { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), text); };
+
+async function open(name: string): Promise<PostgresEngine> {
+ const engine = new PostgresEngine();
+ await engine.connect({ database_url: dbUrl(name), poolSize: 4 });
+ return engine;
+}
+
+async function seed(): Promise<void> {
+ const admin = postgres(adminUrl, { max: 1, prepare: false, onnotice: () => {} });
+ await admin.unsafe('DROP DATABASE IF EXISTS ' + RUN_DB + ' WITH (FORCE)');
+ await admin.unsafe('DROP DATABASE IF EXISTS ' + SEED_DB + ' WITH (FORCE)');
+ await admin.unsafe('CREATE DATABASE ' + SEED_DB);
+ rmSync(home, { recursive: true, force: true }); mkdirSync(root, { recursive: true });
+ git('init', '-q'); git('config', 'user.name', 'Example Writer'); git('config', 'user.email', '[email protected]'); git('config', 'gc.auto', '0');
+ // History: the pages arrive over COMMITS commits; along the way some are rewritten and some renamed (moved, then back).
+ const perCommit = Math.ceil(PAGES / COMMITS);
+ for (let c = 0; c < COMMITS; c++) {
+ for (let i = c * perCommit; i < Math.min(PAGES, (c + 1) * perCommit); i++) write(pathOf(i), contentOf(i));
+ if (c > 2) for (let k = 0; k < 5; k++) { const i = (c * 31 + k * 97) % (c * perCommit); write(pathOf(i), contentOf(i, c)); }
+ if (c > 4 && c % 3 === 0) { const i = (c * 53) % (c * perCommit); const from = join(root, pathOf(i)), to = join(root, 'archive', pathOf(i));
+ if (existsSync(from)) { mkdirSync(dirname(to), { recursive: true }); renameSync(from, to); } }
+ if (c > 6 && c % 3 === 1) { const i = (((c - 1) * 53) % ((c - 1) * perCommit)); const from = join(root, 'archive', pathOf(i)), to = join(root, pathOf(i));
+ if (existsSync(from)) renameSync(from, to); }
+ git('add', '-A'); git('commit', '-qm', 'batch ' + c);
+ }
+ git('gc', '-q');
+ const engine = await open(SEED_DB);
+ try {
+ await engine.initSchema();
+ const { claimWorktree } = await import('../../src/core/persistence/ownership.ts');
+ const { activateSharedSkillPersistence } = await import('../../src/core/persistence/skill-activation.ts');
+ const { performManagedSync } = await import('../../src/core/persistence/sync-run.ts');
+ const { withCoordinatedWrite } = await import('../../src/core/persistence/context.ts');
+ const { disposePersistenceConsumer } = await import('../../src/core/persistence/service.ts');
+ await engine.executeRaw("UPDATE sources SET local_path=$1 WHERE id='default'", [root]);
+ await claimWorktree(engine, 'default', root);
+ await activateSharedSkillPersistence(engine, { confirmQuiesced: true });
+ const started = performance.now();
+ const first = await performManagedSync(engine, { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true });
+ console.error('first sync ' + first.status + ' in ' + Math.round((performance.now() - started) / 1000) + 's');
+ await engine.transaction(tx => withCoordinatedWrite(tx, ['default'], async () => {
+ const ids = await tx.executeRaw<{ id: string }>("UPDATE pages SET type='concept', frontmatter=frontmatter || jsonb_build_object('type','concept','legacy_type',type,'subtype','imported'), "
+ + "knowledge_revision=gen_random_uuid(), revision_write_request_id=NULL WHERE source_id='default' AND id IN (SELECT id FROM pages WHERE source_id='default' ORDER BY slug LIMIT $1) RETURNING id::text", [AHEAD]);
+ await tx.executeRaw("DELETE FROM op_checkpoints WHERE op='sync-import-provenance' AND split_part(fingerprint, ':', 3) = ANY($1::text[])", [ids.map(row => row.id)]);
+ // Pages an older gbrain imported: equal to their files, with no import record.
+ if (LEGACY) await tx.executeRaw("DELETE FROM op_checkpoints WHERE op='sync-import-provenance' AND split_part(fingerprint, ':', 3) IN (SELECT id::text FROM pages WHERE source_id='default' ORDER BY slug DESC LIMIT $1)", [LEGACY]);
+ if (DBONLY) await tx.executeRaw("INSERT INTO pages(source_id,slug,type,title,compiled_truth,timeline,frontmatter,database_only_reason) "
+ + "SELECT 'default','dbonly/p'||lpad(n::text,6,'0'),'note','Database page '||n,'Only in the database '||n||'.','',jsonb_build_object('type','note','title','Database page '||n),'db_only' "
+ + "FROM generate_series(0,$1::int - 1) n", [DBONLY]);
+ }, { requestId: null, principal: { kind: 'application', id: 'probe:fixture' } }));
+ await disposePersistenceConsumer(engine);
+ } finally { await engine.disconnect(); }
+ writeFileSync(join(dir, 'seed.json'), JSON.stringify({ pages: PAGES, ahead: AHEAD, commits: COMMITS, dbonly: DBONLY, kb: KB, legacy: LEGACY, head: git('rev-parse', 'HEAD') }));
+ await admin.end();
+}
+
+const mode = flag('mode', 'convert')!;
+async function run(): Promise<void> {
+ const seeded = JSON.parse(await Bun.file(join(dir, 'seed.json')).text()) as { head: string };
+ const admin = postgres(adminUrl, { max: 1, prepare: false, onnotice: () => {} });
+ await admin.unsafe('DROP DATABASE IF EXISTS ' + RUN_DB + ' WITH (FORCE)');
+ await admin.unsafe('CREATE DATABASE ' + RUN_DB + ' TEMPLATE ' + SEED_DB);
+ await admin.end();
+ git('reset', '-q', '--hard', seeded.head); git('clean', '-qfd');
+ const out = flag('out', join(dir, 'samples-' + mode + '.ndjson'))!;
+ writeFileSync(out, '');
+ const t0 = performance.now();
+ let phase = 'start', scanned = 0, total = 0, ticks = 0;
+ const { heapStats } = await import('bun:jsc');
+ const sample = () => {
+ const m = process.memoryUsage();
+ const types = ticks++ % 10 === 0 ? Object.fromEntries(Object.entries(heapStats().objectTypeCounts).sort((a, b) => b[1] - a[1]).slice(0, 15)) : undefined;
+ appendFileSync(out, JSON.stringify({ t: Math.round((performance.now() - t0) / 100) / 10, phase, scanned, total, rss_mb: Math.round(m.rss / 1048576), heap_mb: Math.round(m.heapUsed / 1048576), ...(types ? { types } : {}) }) + '\n');
+ };
+ const timer = setInterval(sample, 1000);
+ const engine = await open(RUN_DB);
+ try {
+ const { disposePersistenceConsumer } = await import('../../src/core/persistence/service.ts');
+ const onProgress = (event: { phase: string; scanned?: number; total?: number; bankedFiles?: number; round?: unknown }) => { phase = typeof event.round === 'number' ? `${event.phase}:${event.round}` : event.phase; if (event.scanned !== undefined) scanned = event.scanned; else if (event.bankedFiles !== undefined) scanned = event.bankedFiles; if (event.total !== undefined) total = event.total; };
+ if (mode === 'convert') {
+ const { planGitConversion, applyGitConversion } = await import('../../src/core/persistence/git-conversion.ts');
+ phase = 'plan';
+ const { preview } = await planGitConversion(engine, 'default');
+ console.error('plan: adopt ' + preview.adopt.count + ' discard ' + preview.discard.count + ' remaining ' + preview.remaining + ' hash ' + preview.preview_hash);
+ phase = 'apply';
+ const result = await applyGitConversion(engine, 'default', preview.preview_hash, { noEmbed: true, noExtract: !EXTRACT, onProgress } as never);
+ console.error('apply: ' + result.status + ' groups ' + result.committed.groups + ' pages ' + result.committed.pages + ' remaining ' + result.after.remaining);
+ writeFileSync(join(dir, 'result-' + mode + '.json'), JSON.stringify({ preview, result: { ...result, committed: { ...result.committed, commits: result.committed.commits.length } } }, null, 1));
+ } else {
+ const { performManagedSync } = await import('../../src/core/persistence/sync-run.ts');
+ phase = 'sync';
+ const { planGitConversion } = await import('../../src/core/persistence/git-conversion.ts');
+ const { preview } = await planGitConversion(engine, 'default');
+ console.error('plan: pages ' + preview.pages + ' equal ' + preview.equal + ' adopt ' + preview.adopt.count + ' declared ' + preview.declared.count + ' withhold ' + preview.withhold.count + ' discard ' + preview.discard.count);
+ const result = await performManagedSync(engine, { sourceId: 'default', noPull: true, full: args.includes('--full'), noEmbed: true, noExtract: !EXTRACT, onProgress,
+ ...(args.includes('--strict') ? { canonicalOverride: 'git' as const } : {}) } as never);
+ console.error('sync: ' + result.status + ' ' + (result.reason ?? '') + ' added ' + result.added + ' modified ' + result.modified + ' deleted ' + result.deleted);
+ writeFileSync(join(dir, 'result-' + mode + '.json'), JSON.stringify(result, null, 1));
+ }
+ await disposePersistenceConsumer(engine);
+ } finally {
+ clearInterval(timer); sample();
+ await engine.disconnect();
+ }
+ console.error('done in ' + Math.round(performance.now() - t0) / 1000 + 's; samples in ' + out);
+}
+
+
+/** The gbrain config of a CLI run: the database, and embeddings through the stub (`--embed`) or off. */
+function writeConfig(database: string, embed: boolean): void {
+ mkdirSync(join(home, '.gbrain'), { recursive: true });
+ writeFileSync(join(home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: flag('db-url', '') ? dbUrlAt(flag('db-url')!, database) : dbUrl(database),
+ self_upgrade: { mode: 'off' }, ...(embed ? { embedding_model: 'openai:text-embedding-3-small', embedding_dimensions: 1024 } : { embedding_disabled: true }) }, null, 2) + '\n');
+}
+const dbUrlAt = (base: string, name: string) => { const u = new URL(base); u.pathname = '/' + name; return u.toString(); };
+const rssOf = (pid: number) => { try { const m = /VmRSS:\s+(\d+) kB/.exec(readFileSync(`/proc/${pid}/status`, 'utf8')); return m ? Math.round(Number(m[1]) / 1024) : null; } catch { return null; } };
+
+/** Runs the gbrain CLI with the run's home and the stub's environment, sampling its RSS once a second; stdout, stderr and the samples are kept. */
+async function runCli(argv: string[], env: Record<string, string>, name: string, killAfterS?: number): Promise<{ code: number | null; stdout: string; stderr: string; peakMb: number }> {
+ const child = Bun.spawn([process.execPath, join(import.meta.dir, '../../src/cli.ts'), ...argv], { cwd: root, env, stdout: 'pipe', stderr: 'pipe' });
+ const out = join(dir, `samples-${name}.ndjson`);
+ writeFileSync(out, '');
+ const t0 = performance.now();
+ let peakMb = 0, lastProgress = '';
+ const errChunks: string[] = [];
+ const reader = (async () => { for await (const chunk of child.stderr) { const text = new TextDecoder().decode(chunk); errChunks.push(text); const lines = text.split('\n').filter(line => line.startsWith('[sources canonical]') || line.includes('conversion_progress')); if (lines.length) lastProgress = lines.at(-1)!; } })();
+ const timer = setInterval(() => {
+ const rss = rssOf(child.pid);
+ if (rss === null) return;
+ peakMb = Math.max(peakMb, rss);
+ const t = Math.round((performance.now() - t0) / 100) / 10;
+ appendFileSync(out, JSON.stringify({ t, rss_mb: rss, progress: lastProgress }) + '\n');
+ if (killAfterS !== undefined && t >= killAfterS) child.kill('SIGTERM');
+ }, 1000);
+ const stdout = await new Response(child.stdout).text();
+ const code = await child.exited;
+ clearInterval(timer); await reader;
+ const stderr = errChunks.join('');
+ writeFileSync(join(dir, `stderr-${name}.log`), stderr);
+ return { code, stdout, stderr, peakMb };
+}
+
+async function cliRun(): Promise<void> {
+ const embed = args.includes('--embed'), seedEmbed = command === 'embed-seed';
+ const stub = embed || seedEmbed ? await startEmbeddingStub({ latencyMs: Number(flag('latency', '300')), maxConcurrent: Number(flag('concurrency', '2')), dims: 1024 }) : null;
+ const seeded = JSON.parse(await Bun.file(join(dir, 'seed.json')).text()) as { head: string };
+ const target = seedEmbed ? SEED_DB : RUN_DB;
+ if (!seedEmbed) {
+ const admin = postgres(adminUrl, { max: 1, prepare: false, onnotice: () => {} });
+ await admin.unsafe('DROP DATABASE IF EXISTS ' + RUN_DB + ' WITH (FORCE)');
+ await admin.unsafe('CREATE DATABASE ' + RUN_DB + ' TEMPLATE ' + SEED_DB);
+ await admin.end();
+ git('reset', '-q', '--hard', seeded.head); git('clean', '-qfd');
+ // --durable: the box's shape, git durability on with a tracking remote; every Git effect commits and pushes.
+ if (args.includes('--durable')) {
+ const remote = join(dir, 'remote.git');
+ rmSync(remote, { recursive: true, force: true });
+ execFileSync('git', ['init', '-q', '--bare', remote]);
+ try { git('remote', 'remove', 'origin'); } catch { /* none yet */ }
+ git('remote', 'add', 'origin', remote); git('push', '-q', 'origin', 'HEAD:refs/heads/main');
+ git('branch', '--set-upstream-to=origin/main');
+ const engine = await open(RUN_DB);
+ try { console.error('durable bindings: ' + (await engine.executeRaw("UPDATE persistence_host_bindings SET git_durability='enabled' RETURNING git_durability")).length); }
+ finally { await engine.disconnect(); }
+ }
+ }
+ writeConfig(target, embed || seedEmbed);
+ const env: Record<string, string> = { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '', GBRAIN_HOME: home, GBRAIN_MEMORY_TRACE: flag('trace', '10')!,
+ ...(stub ? { OPENAI_API_KEY: 'bench-stub-key', OPENAI_BASE_URL: stub.url } : {}) };
+ try {
+ if (seedEmbed) {
+ const done = await runCli(['embed', '--stale', '--yes', '--json'], env, 'embed-seed');
+ console.error(`embed-seed exit ${done.code} peak ${done.peakMb} MB; stub ${JSON.stringify(stub!.stats)}\n${done.stdout.slice(-600)}`);
+ return;
+ }
+ const argv = mode === 'sync' ? ['sync', '--source', 'default', '--no-pull', ...(args.includes('--full') ? ['--full'] : []), ...(embed ? [] : ['--no-embed']), ...(EXTRACT ? [] : ['--no-extract']), '--json']
+ : null;
+ let hash = '';
+ if (!argv) {
+ const preview = await runCli(['sources', 'canonical', 'default', '--to', 'git', '--dry-run', '--json'], env, 'preview');
+ const parsed = JSON.parse(preview.stdout.trim().split('\n').at(-1)!);
+ hash = parsed.preview_hash;
+ console.error(`preview: adopt ${parsed.adopt.count} declared ${parsed.declared.count} withhold ${parsed.withhold.count} discard ${parsed.discard.count} hash ${hash}`);
+ }
+ const run = await runCli(argv ?? ['sources', 'canonical', 'default', '--to', 'git', '--apply', '--expect', hash, ...(embed ? [] : ['--no-embed']), ...(EXTRACT ? [] : ['--no-extract']), '--json'],
+ env, mode, flag('kill-after') ? Number(flag('kill-after')) : undefined);
+ console.error(`${mode}: exit ${run.code} peak ${run.peakMb} MB${stub ? `; stub ${JSON.stringify(stub.stats)}` : ''}\nstdout tail: ${run.stdout.slice(-1500)}`);
+ } finally { stub?.close(); }
+}
+
+if (!dir) throw new Error('--dir is required');
+if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+if (command === 'seed') await seed();
+else if (command === 'run') await run();
+else if (command === 'cli' || command === 'embed-seed') await cliRun();
+else throw new Error('usage: seed | run');
+process.exit(0);
diff --git a/src/commands/sources-canonical.ts b/src/commands/sources-canonical.ts
index 76f6c08971..1d955309a6 100644
--- a/src/commands/sources-canonical.ts
+++ b/src/commands/sources-canonical.ts
@@ -8,11 +8,16 @@ import type { BrainEngine } from '../core/engine.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import { cliRenderContext, toAgentError } from '../core/agent-output.ts';
+import { registerSignalOwner, triggerCleanupAndExit } from '../core/process-cleanup.ts';
+import { startMemoryTrace } from '../core/memory-trace.ts';
import { applyGitConversion, applyMixedConversion, assertConversionCaller, CONVERSION_GROUP_MAX, planGitConversion, planMixedConversion,
- type GitConversionPreview, type GitConversionResult, type MixedConversionPreview } from '../core/persistence/git-conversion.ts';
+ type ConversionProgress, type GitConversionPreview, type GitConversionResult, type MixedConversionPreview } from '../core/persistence/git-conversion.ts';
const USAGE = 'Usage: gbrain sources canonical <source-id> --to git|mixed [--dry-run | --apply --expect <hash> [--no-embed] [--no-extract]] [--json]';
const DOCS = 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion';
+/** How often a running apply says where it stands (stderr), besides each change of phase. */
+export const PROGRESS_EVERY_MS = 10_000;
+const SIGNAL_EXIT: Record<string, number> = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 };
export function parseCanonicalArgs(args: string[]): { sourceId: string; to: 'git' | 'mixed'; apply: boolean; expect: string | null; json: boolean; noEmbed: boolean; noExtract: boolean } {
const positional: string[] = [];
@@ -65,6 +70,52 @@ function describeMixed(p: MixedConversionPreview): string {
: `Source ${p.source_id} is not git-canonical; there is nothing to change.`;
}
+const progressText = (p: ConversionProgress) =>
+ `[sources canonical] ${p.phase}${p.round ? ` round ${p.round}` : ''}: ${p.scanned}/${p.total} scanned, ${p.groups_committed} group(s) committed`;
+
+/** The source's canonical state as its row says now: `git` once the flag is set, else `mixed`; `unknown` when the read fails or takes over 2 s. */
+async function canonicalState(engine: BrainEngine, sourceId: string): Promise<'git' | 'mixed' | 'unknown'> {
+ const read = engine.executeRaw<{ canonical: string | null }>("SELECT config->>'canonical' AS canonical FROM sources WHERE id=$1", [sourceId])
+ .then(([row]) => row ? (row.canonical === 'git' ? 'git' as const : 'mixed' as const) : 'unknown' as const, () => 'unknown' as const);
+ return Promise.race([read, new Promise<'unknown'>(resolve => setTimeout(() => resolve('unknown'), 2_000).unref?.())]);
+}
+
+/**
+ * #7644: runs an apply that reports where it stands: a stderr line (a JSON line with --json) at each phase and every
+ * PROGRESS_EVERY_MS. SIGTERM, SIGINT or SIGHUP prints a partial report on stdout (its phase, groups committed, items
+ * scanned of the phase's total and the source's canonical state) and exits 128 + the signal. The conversion is crash-safe
+ * on its own; the report only tells the operator where it stopped. GBRAIN_MEMORY_TRACE adds the memory trace (memory-trace.ts).
+ */
+async function withConversionReport<T>(engine: BrainEngine, sourceId: string, json: boolean, apply: (onProgress: (progress: ConversionProgress) => void) => Promise<T>): Promise<T> {
+ let progress: ConversionProgress | null = null, stopping = false;
+ const say = () => { if (progress) process.stderr.write(`${json ? JSON.stringify({ event: 'conversion_progress', source_id: sourceId, ...progress }) : progressText(progress)}\n`); };
+ const onProgress = (next: ConversionProgress) => { const moved = next.phase !== progress?.phase; progress = next; if (moved) say(); };
+ const timer = setInterval(say, PROGRESS_EVERY_MS);
+ timer.unref?.();
+ const stopTrace = startMemoryTrace('sources canonical --apply', () => progress ? { ...progress } : undefined);
+ const interrupted = (signal: NodeJS.Signals) => {
+ if (stopping) return;
+ stopping = true;
+ clearInterval(timer);
+ void (async () => {
+ const canonical = await canonicalState(engine, sourceId);
+ const at = progress ?? { phase: 'adopt', groups_committed: 0, scanned: 0, total: 0 };
+ const error = opError('git_conversion_incomplete', `The conversion of source ${sourceId} was interrupted by ${signal} during ${at.phase}; the source is ${canonical === 'git' ? 'git-canonical' : canonical === 'mixed' ? 'still mixed' : 'in an unknown state'}.`,
+ `Preview again with gbrain sources canonical ${sourceId} --to git --dry-run once gbrain sync status --source ${sourceId} --json shows the source idle, then apply the new hash; the groups already committed read equal.`,
+ { docs: DOCS, detail: 'interrupted' });
+ const report = { status: 'interrupted', signal, source_id: sourceId, ...at, canonical };
+ process.stdout.write(`${json ? JSON.stringify({ ...report, ...toAgentError(error, { transport: 'cli', command: 'sources canonical', render: cliRenderContext() }) })
+ : `Conversion interrupted (${signal}) during ${at.phase}${at.round ? ` round ${at.round}` : ''}: ${at.scanned}/${at.total} scanned, ${at.groups_committed} group(s) committed; source canonical state: ${canonical}.\nNext: ${error.suggestion}`}\n`);
+ await triggerCleanupAndExit(SIGNAL_EXIT[signal] ?? 1);
+ })();
+ };
+ const onSigint = () => interrupted('SIGINT');
+ const release = registerSignalOwner('sources canonical --apply', interrupted);
+ process.on('SIGINT', onSigint);
+ try { return await apply(onProgress); }
+ finally { clearInterval(timer); stopTrace(); release(); process.off('SIGINT', onSigint); }
+}
+
export async function runSourcesCanonical(engine: BrainEngine, args: string[]): Promise<void> {
const json = args.includes('--json');
if (args.includes('--help') || args.includes('-h')) { console.log(`${USAGE}\nPreview first: lists, by category, what converting the source to git-canonical (or back) does (${DOCS}).`); return; }
@@ -79,7 +130,8 @@ export async function runSourcesCanonical(engine: BrainEngine, args: string[]):
return;
}
if (!parsed.apply) { const { preview } = await planGitConversion(engine, parsed.sourceId); print(preview, describeGitPreview(preview)); return; }
- const result = await applyGitConversion(engine, parsed.sourceId, parsed.expect!, { noEmbed: parsed.noEmbed, noExtract: parsed.noExtract });
+ const result = await withConversionReport(engine, parsed.sourceId, json,
+ onProgress => applyGitConversion(engine, parsed.sourceId, parsed.expect!, { noEmbed: parsed.noEmbed, noExtract: parsed.noExtract, onProgress }));
if (result.status === 'incomplete') {
const error = opError('git_conversion_incomplete', `Source ${parsed.sourceId} is git-canonical, and ${result.after.remaining} item(s) were still open after its strict pass.`,
`Writes landed during the conversion. Git wins on the source now; preview again with gbrain sources canonical ${parsed.sourceId} --to git --dry-run and apply the new hash.`,
diff --git a/src/commands/sync.ts b/src/commands/sync.ts
index 0e893801b6..d09f503f18 100644
--- a/src/commands/sync.ts
+++ b/src/commands/sync.ts
@@ -1,4 +1,5 @@
import { type ManagedSyncFailure } from '../core/persistence/sync-failures.ts';
+import { startMemoryTrace } from '../core/memory-trace.ts';
import type { BrainEngine } from '../core/engine.ts';
import { DEFAULT_PARALLEL_SOURCES } from '../core/sync-concurrency.ts';
import { withHumanLogsToStderr, withSourcePrefix } from '../core/console-prefix.ts';
@@ -435,9 +436,8 @@ export async function runSync(engine: BrainEngine, args: string[]) {
// serr/progress are stderr already. The envelope (sync/run.ts emitJson) and
// `sync status|unblock --json` (sync/operator.ts) write through
// writeJsonDocument, the guard's only stdout path.
- return args.includes('--json')
- ? withHumanLogsToStderr(() => runSyncInner(engine, args))
- : runSyncInner(engine, args);
+ const stopTrace = startMemoryTrace('sync');
+ try { return await (args.includes('--json') ? withHumanLogsToStderr(() => runSyncInner(engine, args)) : runSyncInner(engine, args)); } finally { stopTrace(); }
}
/**
diff --git a/src/core/memory-trace.ts b/src/core/memory-trace.ts
new file mode 100644
index 0000000000..9d1e330406
--- /dev/null
+++ b/src/core/memory-trace.ts
@@ -0,0 +1,44 @@
+/**
+ * #7644 / #7596: an opt-in memory trace for long commands (a conversion apply, a sync). With
+ * `GBRAIN_MEMORY_TRACE=<seconds>` set (1 to 3600; any other value is off), the command writes one stderr line every that
+ * many seconds: its label, the seconds since it started, `process.memoryUsage()` in MB (rss, heap used, external,
+ * array buffers), what the caller says it is doing, and under Bun the ten most numerous JavaScriptCore heap object types.
+ * It reads nothing from the brain and writes nothing else, so it is safe to turn on for a run that misbehaves on a host.
+ */
+export const MEMORY_TRACE_ENV = 'GBRAIN_MEMORY_TRACE';
+const TOP_TYPES = 10;
+
+/** The trace interval in ms from the environment, or null when the trace is off. */
+export function memoryTraceIntervalMs(env: NodeJS.ProcessEnv = process.env): number | null {
+ const raw = env[MEMORY_TRACE_ENV]?.trim();
+ if (!raw || !/^\d+$/.test(raw)) return null;
+ const seconds = Number(raw);
+ return seconds >= 1 && seconds <= 3600 ? seconds * 1000 : null;
+}
+
+const mb = (bytes: number) => Math.round(bytes / 1048576);
+
+/** One trace line: memory in MB, the caller's state, and (under Bun) the most numerous heap object types. */
+export async function memoryTraceLine(label: string, startedAt: number, state?: Record<string, unknown>): Promise<string> {
+ const m = process.memoryUsage();
+ let types: Record<string, number> | undefined;
+ if (typeof Bun !== 'undefined') {
+ try {
+ const { heapStats } = await import('bun:jsc');
+ types = Object.fromEntries(Object.entries(heapStats().objectTypeCounts).sort((a, b) => b[1] - a[1]).slice(0, TOP_TYPES));
+ } catch { /* not every runtime exposes heap statistics */ }
+ }
+ return `[gbrain memory] ${JSON.stringify({ label, t_s: Math.round((Date.now() - startedAt) / 1000), rss_mb: mb(m.rss), heap_used_mb: mb(m.heapUsed),
+ external_mb: mb(m.external), array_buffers_mb: mb(m.arrayBuffers), ...(state ? { state } : {}), ...(types ? { types } : {}) })}`;
+}
+
+/** Starts the trace when the environment turns it on; returns the function that stops it (a no-op otherwise). */
+export function startMemoryTrace(label: string, state?: () => Record<string, unknown> | undefined, env: NodeJS.ProcessEnv = process.env,
+ write: (line: string) => void = line => { try { process.stderr.write(line + '\n'); } catch { /* stderr gone */ } }): () => void {
+ const every = memoryTraceIntervalMs(env);
+ if (every === null) return () => undefined;
+ const startedAt = Date.now();
+ const timer = setInterval(() => { void memoryTraceLine(label, startedAt, state?.()).then(write, () => undefined); }, every);
+ timer.unref?.();
+ return () => clearInterval(timer);
+}
diff --git a/src/core/persistence/git-conversion.ts b/src/core/persistence/git-conversion.ts
index c1a42641a0..d2643124ed 100644
--- a/src/core/persistence/git-conversion.ts
+++ b/src/core/persistence/git-conversion.ts
@@ -41,6 +41,7 @@
import { lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, relative, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
+import type { SyncOpts } from '../../commands/sync.ts';
import { loadConfig } from '../config.ts';
import { resolveSourceId } from '../source-resolver.ts';
import { opError, type OperationError } from '../ops/contract.ts';
@@ -75,6 +76,17 @@ const SAMPLE = 20;
const LOCK_WAIT_MS = 30_000;
const MOVED_BATCH = 10_000;
+/** The steps of an apply, in order (`ConversionProgress.phase`). */
+export const CONVERSION_PHASES = ['adopt', 'withhold', 'recheck', 'holds', 'strict_pass_1', 'strict_pass_2'] as const;
+export type ConversionPhase = typeof CONVERSION_PHASES[number];
+/**
+ * Where an apply stands: its phase, the door commits it made (adoption and withholding groups), and the items the phase
+ * went through of its total. In a strict pass, `round` is the pass round, `total` the paths its level read until it
+ * admits, then its batch, and `scanned` the batch entries settled.
+ */
+export interface ConversionProgress { phase: ConversionPhase; groups_committed: number; scanned: number; total: number; round?: number }
+type ConversionReporter = (patch: Partial<ConversionProgress>) => void;
+
export const ADOPT_STATES = ['database_ahead', 'undeclared_database_only', 'worktree_only'] as const;
export type AdoptState = typeof ADOPT_STATES[number];
export const GIT_WINS_REASONS = ['git_ahead', 'deleted_by_commit', 'held_deletion', 'held_drift_unknown'] as const;
@@ -449,14 +461,20 @@ const blockedError = (preview: GitConversionPreview): OperationError => {
};
/** The apply of `--to git` (see the header). Throws preview_changed or git_conversion_blocked before writing anything. */
-export async function applyGitConversion(engine: BrainEngine, sourceId: string, expect: string, opts: { noEmbed?: boolean; noExtract?: boolean } = {}): Promise<GitConversionResult> {
+export async function applyGitConversion(engine: BrainEngine, sourceId: string, expect: string,
+ opts: { noEmbed?: boolean; noExtract?: boolean; onProgress?: (progress: ConversionProgress) => void } = {}): Promise<GitConversionResult> {
const previewCommand = `gbrain sources canonical ${sourceId} --to git --dry-run`;
const plan = await planGitConversion(engine, sourceId);
if (plan.preview.preview_hash !== expect) throw previewChangedError(expect, previewCommand);
if (plan.preview.discard.count) throw blockedError(plan.preview);
- const committed = await adoptGroups(engine, plan, previewCommand);
- const withheld = await withholdGroups(engine, plan, previewCommand);
+ const progress: ConversionProgress = { phase: 'adopt', groups_committed: 0, scanned: 0, total: plan.adopt.length };
+ const report: ConversionReporter = patch => { Object.assign(progress, patch); opts.onProgress?.({ ...progress }); };
+ report({});
+ const committed = await adoptGroups(engine, plan, previewCommand, report);
+ report({ phase: 'withhold', scanned: 0, total: plan.withhold.length });
+ const withheld = await withholdGroups(engine, plan, previewCommand, groups => report({ scanned: progress.scanned + groups.pages, groups_committed: committed.groups + groups.commits }));
const parked = await settleParked(engine, sourceId, plan.parked);
+ report({ phase: 'recheck', scanned: 0, total: 0 });
// The planned holds are cleared only once this check passes, so a refusal keeps them: a held path is judged as the
// strict pass will see it once its hold is cleared. A page the plan settled as equal to its file is judged again once it
// moved since (its equality no longer holds).
@@ -471,20 +489,32 @@ export async function applyGitConversion(engine: BrainEngine, sourceId: string,
if (!result.settled && !result.inFlight && result.gitWins) tally(preview.discard, 'git_overwrites_database', path);
}
if (preview.discard.count) throw blockedError(preview);
+ report({ phase: 'holds', scanned: 0, total: plan.holds.length });
let holdsCleared = 0;
for (const hold of plan.holds) {
if (await engine.transaction(tx => clearGitHold(tx, { sourceId, incarnation: plan.incarnation, path: hold.path, observedAt: hold.observed_at }))) holdsCleared++;
+ report({ scanned: progress.scanned + 1 });
}
const processing = { noEmbed: opts.noEmbed, noExtract: opts.noExtract };
const { performManagedSync } = await import('./sync-run.ts');
- passOrThrow(sourceId, await performManagedSync(engine, { sourceId, noPull: true, full: true, ...processing, canonicalOverride: 'git' }));
+ report({ phase: 'strict_pass_1', scanned: 0, total: 0 });
+ passOrThrow(sourceId, await performManagedSync(engine, { sourceId, noPull: true, full: true, ...processing, canonicalOverride: 'git', onProgress: passProgress(report) }));
await faultPoint('conversion:before_flip', { sourceId });
await recordConversion(engine, sourceId, plan.incarnation, 'git', expect, plan.preview.head);
- passOrThrow(sourceId, await performManagedSync(engine, { sourceId, noPull: true, full: true, ...processing }));
+ report({ phase: 'strict_pass_2', scanned: 0, total: 0, round: undefined });
+ passOrThrow(sourceId, await performManagedSync(engine, { sourceId, noPull: true, full: true, ...processing, onProgress: passProgress(report) }));
const after = (await planGitConversion(engine, sourceId)).preview;
return { status: after.remaining === 0 ? 'converted' : 'incomplete', source_id: sourceId, head: after.head, committed, withheld, holds_cleared: holdsCleared, parked, after };
}
+/** A strict pass's progress events as the apply's: a round's level read sets the total, its batch the scanned entries. */
+function passProgress(report: ConversionReporter): NonNullable<SyncOpts['onProgress']> {
+ return event => {
+ if (event.round) report({ round: event.round.round, scanned: 0, total: event.phase === 'managed_sync.round_admit' ? (event.round.imports ?? 0) + (event.round.deletes ?? 0) : event.round.paths });
+ else if (event.bankedFiles !== undefined && event.phase !== 'managed_sync.level_progress' && event.phase !== 'managed_sync.legacy_swept') report({ scanned: event.bankedFiles, ...(event.total !== undefined ? { total: event.total } : {}) });
+ };
+}
+
/** The paths of `compared` whose page has another revision than the plan compared, or is gone. */
async function movedSincePlan(engine: BrainEngine, sourceId: string, compared: Map<string, PageRevision>): Promise<Set<string>> {
const moved = new Set<string>();
@@ -512,7 +542,7 @@ function passOrThrow(sourceId: string, result: { status: string; reason?: string
* tree moved since the preview stops the apply (preview_changed) before anything else changes: the groups already
* committed hold the bytes the database has, so the rerun preview reads them equal.
*/
-async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string): Promise<GitConversionResult['committed']> {
+async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string, report: ConversionReporter): Promise<GitConversionResult['committed']> {
const { context, adopt } = plan;
const root = context.root;
const commits: string[] = [];
@@ -535,6 +565,7 @@ async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, preview
const commit = await commitThroughDoor(root, files.map(file => ({ path: file.path })), lock);
if (commit) commits.push(commit);
} finally { await lock.release(); }
+ report({ scanned: start + group.length, groups_committed: commits.length });
await faultPoint('conversion:group_committed', { sourceId: context.sourceId, round: start / CONVERSION_GROUP_MAX + 1 });
}
return { groups: commits.length, pages: adopt.length, commits };
@@ -548,7 +579,8 @@ async function adoptGroups(engine: BrainEngine, plan: GitConversionPlan, preview
* is already absent from HEAD only gets the stamp. A page whose revision or working tree moved since the preview stops
* the apply (preview_changed); after a crash between a commit and its stamp, the rerun preview lists the page as file_absent.
*/
-async function withholdGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string): Promise<GitConversionResult['withheld']> {
+async function withholdGroups(engine: BrainEngine, plan: GitConversionPlan, previewCommand: string,
+ onGroup: (done: { pages: number; commits: number }) => void): Promise<GitConversionResult['withheld']> {
const { context } = plan;
const root = context.root;
const commits: string[] = [];
@@ -580,10 +612,11 @@ async function withholdGroups(engine: BrainEngine, plan: GitConversionPlan, prev
if (commit) commits.push(commit);
await stamp(group);
} finally { await lock.release(); }
+ onGroup({ pages: group.length, commits: commits.length });
}
const absent = plan.withhold.filter(item => item.kind === 'file_absent');
for (const item of absent) await assertUnchanged(item);
- if (absent.length) await stamp(absent);
+ if (absent.length) { await stamp(absent); onGroup({ pages: absent.length, commits: commits.length }); }
return { pages: plan.withhold.length, commits };
}
diff --git a/test/git-canonical-conversion-progress.test.ts b/test/git-canonical-conversion-progress.test.ts
new file mode 100644
index 0000000000..da8f8e8ecb
--- /dev/null
+++ b/test/git-canonical-conversion-progress.test.ts
@@ -0,0 +1,153 @@
+/**
+ * #7644: a conversion apply says where it stands. \`applyGitConversion\` reports each phase in order with the door groups it
+ * committed and the items the phase went through; the CLI prints them on stderr (a JSON line with --json), and a SIGTERM
+ * during the strict pass prints a partial report on stdout naming the phase, the groups committed, the paths scanned and
+ * the source's canonical state, then exits 143. GBRAIN_MEMORY_TRACE adds a periodic memory line.
+ */
+import { afterAll, describe, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { withCoordinatedWrite } from '../src/core/persistence/context.ts';
+import { performManagedSync } from '../src/core/persistence/sync-run.ts';
+import { applyGitConversion, CONVERSION_GROUP_MAX, CONVERSION_PHASES, planGitConversion, type ConversionProgress } from '../src/core/persistence/git-conversion.ts';
+import { runSourcesCanonical } from '../src/commands/sources-canonical.ts';
+import { _resetCliExitVerdictForTests } from '../src/core/cli-force-exit.ts';
+import { memoryTraceIntervalMs, memoryTraceLine, startMemoryTrace } from '../src/core/memory-trace.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { TEST_WRITE_ATTRIBUTION } from './helpers/write-attribution.ts';
+
+const REPO = join(import.meta.dir, '..');
+const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, '-c', 'commit.gpgsign=false', ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const note = (slug: string, body: string) => '---\ntype: concept\ntitle: ' + slug + '\n---\n\n' + body + '\n';
+const OPTS = { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true };
+const BULK = CONVERSION_GROUP_MAX + 1;
+const bulkSetup = ({ root }: { root: string }) => {
+ mkdirSync(join(root, 'bulk'), { recursive: true }); git(root, 'init', '-q');
+ git(root, 'config', 'user.name', 'Example Writer'); git(root, 'config', 'user.email', '[email protected]');
+ for (let i = 0; i < BULK; i++) writeFileSync(join(root, 'bulk/p' + String(i).padStart(3, '0') + '.md'), note('bulk/p' + i, 'Committed body ' + i + '.'));
+ git(root, 'add', '-A'); git(root, 'commit', '-qm', 'bulk');
+};
+/** Every bulk page edited in the database alone: the plan adopts them in two door groups. */
+const editBulk = (engine: Parameters<Parameters<typeof managedBrain>[0]>[0]['engine']) => engine.transaction(tx => withCoordinatedWrite(tx, ['default'], () => tx.executeRaw(
+ "UPDATE pages SET compiled_truth=compiled_truth || ' Edited in the database.', knowledge_revision=gen_random_uuid(), revision_write_request_id=NULL WHERE source_id='default' AND slug LIKE 'bulk/%'"), TEST_WRITE_ATTRIBUTION));
+
+for (const backend of testBackends()) describe('[' + backend + '] #7644: a conversion apply reports its progress', () => {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+
+ test('the apply reports every phase in order, the door groups as they commit, and each strict pass round', () => managedBrain(async ({ engine }) => {
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ await editBulk(engine);
+ const { preview } = await planGitConversion(engine, 'default');
+ const seen: ConversionProgress[] = [];
+ expect(await applyGitConversion(engine, 'default', preview.preview_hash, { noEmbed: true, noExtract: true, onProgress: progress => seen.push(progress) }))
+ .toMatchObject({ status: 'converted', committed: { groups: 2, pages: BULK } });
+ expect([...new Set(seen.map(progress => progress.phase))]).toEqual([...CONVERSION_PHASES]);
+ // Phases only move forward.
+ expect(seen.map(progress => CONVERSION_PHASES.indexOf(progress.phase))).toEqual(seen.map(progress => CONVERSION_PHASES.indexOf(progress.phase)).sort((a, b) => a - b));
+ expect(seen.filter(progress => progress.phase === 'adopt').map(({ scanned, total, groups_committed }) => [scanned, total, groups_committed]))
+ .toEqual([[0, BULK, 0], [CONVERSION_GROUP_MAX, BULK, 1], [BULK, BULK, 2]]);
+ for (const phase of ['strict_pass_1', 'strict_pass_2'] as const) {
+ const pass = seen.filter(progress => progress.phase === phase);
+ expect(pass.every(progress => progress.groups_committed === 2)).toBe(true);
+ // A round's level read sets the total, and its batch entries settle up to it.
+ expect(pass.some(progress => progress.round === 1 && progress.total >= BULK)).toBe(true);
+ expect(pass.some(progress => progress.round === 1 && progress.scanned > 0 && progress.scanned <= progress.total)).toBe(true);
+ }
+ }, { databaseUrl, setup: bulkSetup }), 300_000);
+
+ test('the CLI prints a progress line on stderr at each phase, and a JSON line with --json', () => managedBrain(async ({ engine }) => {
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ await editBulk(engine);
+ const { preview } = await planGitConversion(engine, 'default');
+ const err: string[] = [], out: string[] = [];
+ const write = process.stderr.write, log = console.log, priorExitCode = process.exitCode;
+ process.stderr.write = ((chunk: string | Uint8Array) => { err.push(String(chunk)); return true; }) as typeof process.stderr.write;
+ console.log = (line: unknown) => { out.push(String(line)); };
+ try { await runSourcesCanonical(engine, ['default', '--to', 'git', '--apply', '--expect', preview.preview_hash, '--no-embed', '--no-extract', '--json']); }
+ finally { process.stderr.write = write; console.log = log; _resetCliExitVerdictForTests(); process.exitCode = priorExitCode ?? 0; }
+ expect(JSON.parse(out.at(-1)!)).toMatchObject({ status: 'converted' });
+ const lines = err.join('').split('\n').filter(line => line.includes('"conversion_progress"')).map(line => JSON.parse(line));
+ expect(lines.map(line => line.phase)).toEqual([...CONVERSION_PHASES]);
+ expect(lines.at(-1)).toMatchObject({ event: 'conversion_progress', source_id: 'default', phase: 'strict_pass_2', groups_committed: 2 });
+ }, { databaseUrl, setup: bulkSetup }), 300_000);
+});
+
+describe('#7644: SIGTERM during the strict pass', () => {
+ const work = mkdtempSync(join(tmpdir(), 'gbrain-conversion-sigterm-'));
+ afterAll(() => rmSync(work, { recursive: true, force: true }));
+ const env: Record<string, string> = Object.fromEntries(Object.entries(process.env)
+ .filter(([key, value]) => value !== undefined && !['DATABASE_URL', 'GBRAIN_DATABASE_URL', 'GBRAIN_SOURCE', 'GBRAIN_BRAIN_ID', 'GBRAIN_MEMORY_TRACE'].includes(key))) as Record<string, string>;
+ env.GBRAIN_HOME = join(work, 'home');
+ const cli = (...args: string[]) => {
+ const result = spawnSync(process.execPath, ['--no-env-file', join(REPO, 'src', 'cli.ts'), ...args], { cwd: REPO, env, encoding: 'utf8', timeout: 180_000 });
+ if (result.status !== 0) throw new Error('gbrain ' + args.join(' ') + ' exited ' + result.status + ':\n' + result.stdout + result.stderr);
+ return result.stdout;
+ };
+
+ test('prints a partial report naming the phase, groups, paths scanned and canonical state, then exits 143', async () => {
+ const checkout = join(work, 'notes');
+ mkdirSync(join(checkout, 'bulk'), { recursive: true });
+ git(checkout, 'init', '-q'); git(checkout, 'config', 'user.name', 'Example Writer'); git(checkout, 'config', 'user.email', '[email protected]');
+ for (let i = 0; i < 300; i++) writeFileSync(join(checkout, 'bulk/p' + i + '.md'), note('bulk/p' + i, 'Committed body ' + i + '.'));
+ git(checkout, 'add', '-A'); git(checkout, 'commit', '-qm', 'bulk');
+ cli('init', '--pglite', '--no-embedding');
+ cli('config', 'set', 'self_upgrade.mode', 'off');
+ cli('sources', 'add', 'notes', '--path', checkout);
+ cli('sync', '--source', 'notes', '--no-pull', '--no-embed');
+ const hash = JSON.parse(cli('sources', 'canonical', 'notes', '--to', 'git', '--dry-run', '--json').trim().split('\n').at(-1)!).preview_hash as string;
+
+ const child = Bun.spawn([process.execPath, '--no-env-file', join(REPO, 'src', 'cli.ts'), 'sources', 'canonical', 'notes', '--to', 'git', '--apply', '--expect', hash, '--no-embed', '--json'],
+ { cwd: REPO, env, stdout: 'pipe', stderr: 'pipe' });
+ let err = '', killed = false;
+ const reading = (async () => {
+ for await (const chunk of child.stderr) {
+ err += new TextDecoder().decode(chunk);
+ if (!killed && err.includes('"phase":"strict_pass_1"')) { killed = true; child.kill('SIGTERM'); }
+ }
+ })();
+ const stdout = await new Response(child.stdout).text();
+ const code = await child.exited;
+ await reading;
+ expect(killed).toBe(true);
+ expect(code).toBe(143);
+ const report = JSON.parse(stdout.trim().split('\n').at(-1)!);
+ expect(report).toMatchObject({ status: 'interrupted', signal: 'SIGTERM', source_id: 'notes', phase: 'strict_pass_1', groups_committed: 0, canonical: 'mixed',
+ code: 'git_conversion_incomplete', detail: 'interrupted' });
+ expect(report.scanned).toBeLessThanOrEqual(report.total);
+ // The conversion is crash-safe: the source stays mixed, and a new preview applies.
+ expect(JSON.parse(cli('sources', 'canonical', 'notes', '--to', 'git', '--dry-run', '--json').trim().split('\n').at(-1)!)).toMatchObject({ strict: false, discard: { count: 0 } });
+ }, 300_000);
+});
+
+describe('#7644: GBRAIN_MEMORY_TRACE', () => {
+ test('is off unless set to 1..3600 seconds', () => {
+ expect(memoryTraceIntervalMs({})).toBeNull();
+ for (const value of ['', '0', '-5', 'x', '1.5', '3601']) expect(memoryTraceIntervalMs({ GBRAIN_MEMORY_TRACE: value })).toBeNull();
+ expect(memoryTraceIntervalMs({ GBRAIN_MEMORY_TRACE: '10' })).toBe(10_000);
+ });
+
+ test('writes the label, memory in MB, the caller state and the most numerous heap object types', async () => {
+ const line = await memoryTraceLine('sources canonical --apply', Date.now(), { phase: 'strict_pass_1', scanned: 3, total: 9 });
+ expect(line.startsWith('[gbrain memory] ')).toBe(true);
+ const value = JSON.parse(line.slice('[gbrain memory] '.length));
+ expect(value).toMatchObject({ label: 'sources canonical --apply', state: { phase: 'strict_pass_1', scanned: 3, total: 9 } });
+ for (const key of ['rss_mb', 'heap_used_mb', 'external_mb', 'array_buffers_mb']) expect(typeof value[key]).toBe('number');
+ expect(Object.keys(value.types).length).toBeGreaterThan(0);
+ });
+
+ test('emits on its interval only while it runs', async () => {
+ const lines: string[] = [];
+ expect(startMemoryTrace('off', undefined, {}, line => lines.push(line))()).toBeUndefined();
+ const stop = startMemoryTrace('sync', () => ({ round: 1 }), { GBRAIN_MEMORY_TRACE: '1' }, line => lines.push(line));
+ await Bun.sleep(2_300);
+ stop();
+ const seen = lines.length;
+ expect(seen).toBeGreaterThanOrEqual(1);
+ expect(lines.every(line => line.includes('"label":"sync"') && line.includes('"state":{"round":1}'))).toBe(true);
+ await Bun.sleep(1_200);
+ expect(lines.length).toBe(seen);
+ });
+});
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 98b0829cbc..12a830bc82 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -82,6 +82,7 @@ test/forecast-journal.test.ts
test/frontmatter-reserved-keys-6454.test.ts
test/get-chunks-paging.test.ts
test/git-canonical-conversion-g8a.test.ts
+test/git-canonical-conversion-progress.test.ts
test/git-canonical-declared-g7a.test.ts
test/git-canonical-indexer-g3.test.ts
test/git-door-plumbing-gbra133.test.ts