Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.20.0 Cathedral II Layer 8 D3 — reconcile-links batch command.
*
* Closes the v0.19.0 Layer 6 doc↔impl order-dependency bug. When a
* markdown guide cites `src/core/sync.ts:42` but the code source
* hasn't been synced yet, the forward-scan at import time inserts
* nothing because `addLink`'s inner SELECT drops edges to missing
* pages. The guide and the code eventually both exist, but the edge
* never materialized.
*
* D3 fixes this batch-style: walk every markdown page, re-run
* `extractCodeRefs`, and call `addLink(md, code, ..., 'documents')` +
* reverse for each hit. ON CONFLICT DO NOTHING on the `links` table
* makes the operation idempotent — edges that already exist stay,
* new edges land.
*
* Why batch over per-import-reverse-scan: codex 2-phase review
* flagged the per-import approach as O(N) ILIKE/JOIN queries per
* code file imported. On a 47K-page brain first-syncing 5K code
* files, that's 5K ILIKE scans. A user-triggered batch pass on an
* already-synced brain is one walk, fully indexed via the existing
* slug lookup in addLink.
*
* A segmented page (Wave B) is read block by block (`bodyLines`): a code ref
* never spans a line, so it gets the refs the same text held as one string
* gets. A page a block read refuses is listed in `skipped` and the walk goes on.
*/
import type { BrainEngine } from '../core/engine.ts';
import { PageMissingError } from '../core/engine-errors.ts';
import type { Page } from '../core/types.ts';
import { extractCodeRefs, type CodeRef } from '../core/link-extraction.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
import { bodyLines } from '../core/page-body/segment-lines.ts';
import { slugifyCodePath } from '../core/sync.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { ALL_SOURCES, resolveSourceId } from '../core/source-resolver.ts';
import { flagValueError } from '../cli/flag-values.ts';
export interface ReconcileLinksResult {
status: 'ok' | 'auto_link_disabled';
markdownPagesScanned: number;
codeRefsFound: number;
edgesAttempted: number;
edgesTargetsMissing: number;
/** Pages a per-page refusal kept out of the scan (present only when there are any). */
skipped?: Array<{ slug: string; reason: string; message: string }>;
}
/**
* Code refs of a page's `${compiled_truth}\n${timeline}`, first occurrence of each path, in text order. A segmented
* page is read a block of lines at a time; `index` stays an offset into the joined text.
*/
async function pageCodeRefs(engine: BrainEngine, page: Page): Promise<CodeRef[]> {
if (!isSegmentedPage(page)) return extractCodeRefs((page.compiled_truth || '') + '\n' + (page.timeline || ''));
const seen = new Set<string>();
const refs: CodeRef[] = [];
let at = 0;
for await (const lines of bodyLines(engine, page)) {
const text = lines.join('\n');
for (const ref of extractCodeRefs(text)) {
if (seen.has(ref.path)) continue;
seen.add(ref.path);
refs.push({ path: Buffer.from(ref.path, 'utf8').toString('utf8'), line: ref.line, index: ref.index + at });
}
at += text.length + 1;
}
return refs;
}
export interface ReconcileLinksOpts {
/** The one source whose pages are scanned and linked; defaults to `default`, where addLink writes when none is named. */
sourceId?: string;
dryRun?: boolean;
}
/**
* Scan one source's markdown pages for code-path references (e.g.
* `src/core/sync.ts`, `lib/foo.py:42`) and create bidirectional
* doc↔impl edges (`documents` + `documented_by`) for each hit
* that resolves to a code page. Idempotent via ON CONFLICT DO
* NOTHING in the underlying addLink path.
*
* Called by `gbrain reconcile-links` CLI surface. Respects the
* `auto_link` config: if the user has disabled auto-linking on
* put_page, reconcile-links doesn't silently re-populate those
* edges either.
*/
export async function runReconcileLinks(
engine: BrainEngine,
opts: ReconcileLinksOpts = {},
): Promise<ReconcileLinksResult> {
const sourceId = opts.sourceId ?? 'default';
if (sourceId === ALL_SOURCES) {
throw opError('invalid_params', `reconcile-links links pages inside one source; ${ALL_SOURCES} is not a source.`,
'Name one source with --source (gbrain sources list shows the ids).');
}
// Respect auto_link config (same gate put_page uses). A user that
// explicitly turned off auto-link doesn't want reconcile-links
// writing edges back either.
const autoLinkCfg = await engine.getConfig('auto_link');
if (autoLinkCfg === 'false') {
return {
status: 'auto_link_disabled',
markdownPagesScanned: 0,
codeRefsFound: 0,
edgesAttempted: 0,
edgesTargetsMissing: 0,
};
}
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
// Walk the source's markdown slugs. listPages(markdown-only filter) isn't exposed,
// so filter at call time via page_kind. Not using getAllSlugs because we
// also need compiled_truth + timeline for extractCodeRefs.
const mdSlugs = (await engine.executeRaw<{ slug: string }>(
`SELECT slug FROM pages WHERE page_kind = 'markdown' AND source_id = $1 ORDER BY slug`,
[sourceId],
)).map(r => r.slug);
progress.start('reconcile_links.scan', mdSlugs.length);
let codeRefsFound = 0;
let edgesAttempted = 0;
let edgesTargetsMissing = 0;
const skipped: NonNullable<ReconcileLinksResult['skipped']> = [];
// Fetch pages one at a time via getPage (no bulk read helper exists yet).
// On a 47K-page brain this is the slow path; a v0.20.x follow-up can add
// getPagesBatch. For the typical 2K–5K markdown count it's fine.
// v0.18.0+ multi-source: source-scope getPage so reconcile picks up the
// intended-source row for `default`-vs-`<source>` ambiguity. The link
// edges below also propagate the same sourceId (Data R1 MED 1: opt was
// declared on ReconcileLinksOpts but ignored end-to-end).
const getPageOpts = { sourceId };
const linkOpts = { fromSourceId: sourceId, toSourceId: sourceId, originSourceId: sourceId };
for (const mdSlug of mdSlugs) {
const page = await engine.getPage(mdSlug, getPageOpts);
if (!page) {
progress.tick(1, mdSlug);
continue;
}
let refs: CodeRef[];
try {
refs = await pageCodeRefs(engine, page);
} catch (e: unknown) {
if (!(e instanceof OperationError && e.code === 'page_body_segmented')) throw e;
skipped.push({ slug: mdSlug, reason: e.code, message: e.message });
progress.tick(1, `${mdSlug} (skipped)`);
continue;
}
if (refs.length === 0) {
progress.tick(1, mdSlug);
continue;
}
codeRefsFound += refs.length;
if (opts.dryRun) {
progress.tick(1, `${mdSlug} (+${refs.length} refs)`);
continue;
}
for (const ref of refs) {
const codeSlug = slugifyCodePath(ref.path);
const ctx = ref.line ? `cited at ${ref.path}:${ref.line}` : ref.path;
edgesAttempted++;
try {
// Forward: guide documents code. addLink's inner JOIN drops silently
// if codeSlug isn't a page yet (benign — counted below). Source-
// qualified per opts.sourceId; same-source assumption mirrors the
// import-file.ts:303 doc↔impl auto-link.
await engine.addLink(mdSlug, codeSlug, ctx, 'documents', 'markdown', mdSlug, 'compiled_truth', linkOpts); // gbrain-allow-direct-insert: gbrain reconcile-links command — code-graph reconciliation from markdown references
await engine.addLink(codeSlug, mdSlug, ref.path, 'documented_by', 'markdown', mdSlug, 'compiled_truth', linkOpts); // gbrain-allow-direct-insert: gbrain reconcile-links command — reverse documented_by edge
} catch (e: unknown) {
// A missing code page is expected and counted; any other write failure fails the run.
if (!(e instanceof PageMissingError)) {
progress.finish();
throw e;
}
edgesTargetsMissing++;
}
}
progress.tick(1, `${mdSlug} (+${refs.length} refs)`);
}
progress.finish();
return {
status: 'ok',
markdownPagesScanned: mdSlugs.length,
codeRefsFound,
edgesAttempted,
edgesTargetsMissing,
...(skipped.length ? { skipped } : {}),
};
}
/**
* CLI entry. Parses argv, runs runReconcileLinks, prints a summary.
* --source <id> picks the source (else the usual chain: GBRAIN_SOURCE,
* .gbrain-source, registered path, brain default). --dry-run reports counts
* without writing. --json emits machine output.
*/
export async function runReconcileLinksCli(engine: BrainEngine, args: string[]): Promise<void> {
const dryRun = args.includes('--dry-run');
const jsonOut = args.includes('--json');
const sourceAt = args.indexOf('--source');
const explicit = sourceAt === -1 ? null : args[sourceAt + 1];
if (explicit !== null && (!explicit || explicit.startsWith('-'))) throw flagValueError('--source', explicit, 'a source id', 'default');
const sourceId = await resolveSourceId(engine, explicit);
const result = await runReconcileLinks(engine, { dryRun, sourceId });
if (jsonOut) {
console.log(JSON.stringify(result));
return;
}
if (result.status === 'auto_link_disabled') {
console.log(
'[reconcile-links] auto_link is disabled in config; skipping. ' +
'Set `gbrain config set auto_link true` to re-enable.',
);
return;
}
const header = dryRun ? 'reconcile-links (dry run)' : 'reconcile-links';
console.log(
`${header}: scanned ${result.markdownPagesScanned} markdown pages, ` +
`found ${result.codeRefsFound} code refs, ` +
`attempted ${result.edgesAttempted} edges` +
(result.edgesTargetsMissing > 0
? ` (${result.edgesTargetsMissing} targets missing code page)`
: ''),
);
for (const page of result.skipped ?? []) console.log(`[reconcile-links] skipped ${page.slug} (${page.reason}): ${page.message}`);
}