Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* gbrain integrity — scan, report, and repair brain-integrity issues.
*
* The user-visible shipping milestone for the Knowledge Runtime delta.
* Uses PR 1's resolver SDK + PR 2's BrainWriter to target two known pain
* points quantified in brain/CITATIONS.md:
*
* 1. Bare tweet references: "Garry tweeted about X" with no URL
* (CITATIONS.md: 1,424 out of 3,115 people pages)
* 2. Dead or rotted URLs in existing citations
*
* Subcommands:
* gbrain integrity check Read-only report to stdout
* gbrain integrity auto Three-bucket repair with confidence
* gbrain integrity auto --dry-run Same as auto, no writes
*
* Three-bucket confidence (contract with x_handle_to_tweet resolver):
* >= 0.8 → auto-repair through BrainWriter transaction
* 0.5–0.8 → append to ~/.gbrain/integrity-review.md for human review
* < 0.5 → skip, log to ~/.gbrain/integrity.log.jsonl
*
* Progress is durable at ~/.gbrain/integrity-progress.jsonl. Re-running
* after a kill resumes from the last processed slug; already-repaired pages
* are not revisited.
*/
import { appendFileSync, existsSync, readFileSync, mkdirSync, writeFileSync } from 'fs';
import { dirname } from 'path';
import { loadConfig, toEngineConfig, gbrainPath } from '../core/config.ts';
import { createEngine } from '../core/engine-factory.ts';
import type { BrainEngine } from '../core/engine.ts';
import { BrainWriter } from '../core/output/writer.ts';
import {
getDefaultRegistry,
type ResolverContext,
type ResolverResult,
} from '../core/resolvers/index.ts';
import { registerBuiltinResolvers } from './resolvers.ts';
import { tweetCitation } from '../core/output/scaffold.ts';
import { INTEGRITY_SUBCOMMANDS, ROUTERS, subcommandHelpRequested } from '../cli/subcommands.ts';
import { OperationError } from '../core/ops/contract.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
import type { BodyStorage } from '../core/page-body/types.ts';
import { bodyLines } from '../core/page-body/segment-lines.ts';
import { detachedValue } from '../core/page-body/segment-blocks.ts';
import { HANDLE_FRONTMATTER_KEYS } from '../core/by-mention.ts';
// ---------------------------------------------------------------------------
// Paths
// ---------------------------------------------------------------------------
// Lazy: GBRAIN_HOME may be set after module load.
const getReviewFile = () => gbrainPath('integrity-review.md');
const getLogFile = () => gbrainPath('integrity.log.jsonl');
const getProgressFile = () => gbrainPath('integrity-progress.jsonl');
// ---------------------------------------------------------------------------
// Bare-tweet detection
// ---------------------------------------------------------------------------
/**
* Phrases that plausibly reference a tweet without actually linking to one.
* Case-insensitive. We explicitly REQUIRE an X handle on the page (via
* frontmatter.x_handle or inline @handle) before repair — otherwise there's
* no seed to search from and confidence would be zero.
*/
const BARE_TWEET_PHRASES = [
/\btweeted about\b/i,
/\bin (?:a |the )?(?:recent |viral )?tweet\b/i,
/\bon (?:a |the )?(?:recent |viral )?tweet\b/i,
/\bwrote (?:a |the )?(?:tweet|post)\b/i,
/\bposted on X\b/i,
/\bvia X\b(?!\s*\/)/i, // "via X" but not "via X/handle" (already cited)
/\bhis (?:recent |)tweet\b/i,
/\bher (?:recent |)tweet\b/i,
/\btheir (?:recent |)tweet\b/i,
];
const URL_NEARBY_RE = /https?:\/\/(?:x\.com|twitter\.com)\/[A-Za-z0-9_]+\/status\/\d+/;
export interface BareTweetHit {
slug: string;
line: number;
rawLine: string;
phrase: string;
}
interface FenceState { inside: boolean; marker: string; line: number }
const fenceState = (): FenceState => ({ inside: false, marker: '', line: 0 });
/** Calls `visit` with each line outside a ``` or ~~~ fence and its 1-based number; `state` carries across calls, so lines may arrive in runs. */
function unfencedLines(lines: readonly string[], state: FenceState, visit: (line: string, n: number) => void): void {
for (const line of lines) {
state.line++;
if (state.inside) {
if (line.startsWith(state.marker)) state.inside = false;
continue;
}
if (line.startsWith('```') || line.startsWith('~~~')) {
state.inside = true;
state.marker = line.startsWith('```') ? '```' : '~~~';
continue;
}
visit(line, state.line);
}
}
function bareTweetHit(line: string, n: number, slug: string): BareTweetHit | null {
// If the line already contains a tweet URL, it's cited — skip
if (URL_NEARBY_RE.test(line)) return null;
// If the line carries an explicit source citation (e.g.
// "[Source: X, @handle, 2026-05-28]"), it's already attributed — skip.
// Catches instructional/example lines in recipe docs that demonstrate
// the CORRECT citation format. (v0.42.x)
if (/\[\s*source:/i.test(line)) return null;
// Strip inline-code spans (`...`) before matching: phrases shown as
// inline-code templates in docs are examples, not bare claims. The
// fenced-code skip above only covers ``` blocks, not inline backticks.
const lineForMatch = line.replace(/`[^`]*`/g, '');
for (const re of BARE_TWEET_PHRASES) {
const m = lineForMatch.match(re);
if (m) return { slug, line: n, rawLine: line.trim(), phrase: m[0] }; // one finding per line is enough
}
return null;
}
export function findBareTweetHits(compiledTruth: string, slug: string): BareTweetHit[] {
const hits: BareTweetHit[] = [];
unfencedLines(compiledTruth.split('\n'), fenceState(), (line, n) => {
const hit = bareTweetHit(line, n, slug);
if (hit) hits.push(hit);
});
return hits;
}
// ---------------------------------------------------------------------------
// Dead-link detection
// ---------------------------------------------------------------------------
const MD_LINK_EXTERNAL_RE = /\[[^\]]+\]\((https?:\/\/[^)]+)\)/g;
export interface ExternalLinkHit {
slug: string;
line: number;
url: string;
}
function externalLinkHits(line: string, n: number, slug: string, hits: ExternalLinkHit[]): void {
MD_LINK_EXTERNAL_RE.lastIndex = 0;
let m: RegExpExecArray | null;
while ((m = MD_LINK_EXTERNAL_RE.exec(line)) !== null) {
hits.push({ slug, line: n, url: m[1] });
}
}
export function findExternalLinks(compiledTruth: string, slug: string): ExternalLinkHit[] {
const hits: ExternalLinkHit[] = [];
unfencedLines(compiledTruth.split('\n'), fenceState(), (line, n) => externalLinkHits(line, n, slug, hits));
return hits;
}
/** A page the scan could not read; the run goes on without it. */
export interface IntegritySkippedPage { slug: string; source_id: string; reason: string; message: string }
/**
* Bare-tweet and external-link hits of a page's compiled truth. A segmented
* page (Wave B) is read block by block (`bodyLines` over its truth region),
* with fence state and line numbers carried across blocks, so it gets the
* hits the same text held as one string gets.
*/
async function integrityHitsOf(engine: BrainEngine, page: { id: number; slug: string; body_storage?: BodyStorage | null; compiled_truth?: string | null },
sourceId: string): Promise<{ bare: BareTweetHit[]; external: ExternalLinkHit[] }> {
if (!isSegmentedPage(page)) {
const text = page.compiled_truth ?? '';
return { bare: findBareTweetHits(text, page.slug), external: findExternalLinks(text, page.slug) };
}
const bare: BareTweetHit[] = [];
const external: ExternalLinkHit[] = [];
const bareState = fenceState(), externalState = fenceState();
for await (const lines of bodyLines(engine, { id: Number(page.id), slug: page.slug, source_id: sourceId }, { regions: ['truth'] })) {
unfencedLines(lines, bareState, (line, n) => {
const hit = bareTweetHit(line, n, page.slug);
if (hit) bare.push(detachedValue(hit));
});
unfencedLines(lines, externalState, (line, n) => {
const found: ExternalLinkHit[] = [];
externalLinkHits(line, n, page.slug, found);
external.push(...found.map(detachedValue));
});
}
return { bare, external };
}
/** A per-page refusal (a segmented page whose run is too long to read as one block) becomes a skipped page; anything else propagates. */
function skippedPage(error: unknown, page: { slug: string; source_id: string }): IntegritySkippedPage {
if (error instanceof OperationError && error.code === 'page_body_segmented') return { slug: page.slug, source_id: page.source_id, reason: error.code, message: error.message };
throw error;
}
// ---------------------------------------------------------------------------
// Progress tracking
// ---------------------------------------------------------------------------
interface ProgressEntry {
slug: string;
/**
* Source the row belongs to. Progress used to be keyed by slug alone, so a
* resume SKIPPED same-slug pages in every other source (the scan iterates
* (slug, source_id) pairs). Legacy entries without source_id are treated as
* default-source only.
*/
source_id?: string;
status: 'repaired' | 'reviewed' | 'skipped' | 'error';
timestamp: string;
}
/** Composite progress key — (source, slug), tab-separated (tabs can't appear in either). */
function progressKey(sourceId: string | undefined, slug: string): string {
return `${sourceId ?? 'default'}\t${slug}`;
}
function loadProgress(): Set<string> {
if (!existsSync(getProgressFile())) return new Set();
const seen = new Set<string>();
const content = readFileSync(getProgressFile(), 'utf-8');
let legacy = 0;
for (const line of content.split('\n')) {
if (!line.trim()) continue;
try {
const entry = JSON.parse(line) as ProgressEntry;
if (entry.source_id == null) legacy++;
seen.add(progressKey(entry.source_id, entry.slug));
} catch {
/* skip malformed lines */
}
}
if (legacy > 0) {
// Pre-(source_id, slug) ledger entries key as default-source only, so a
// resume re-scans non-default-source pages they may have covered. Say so
// once — a silent partial re-scan reads as "resume is broken".
console.error(
`integrity: ${legacy} resume-ledger entr${legacy === 1 ? 'y' : 'ies'} predate source tracking; ` +
`matching them to the default source only (non-default-source pages re-scan — idempotent, just slower).`,
);
}
return seen;
}
function appendProgress(entry: ProgressEntry): void {
ensureDir(getProgressFile());
appendFileSync(getProgressFile(), JSON.stringify(entry) + '\n', 'utf-8');
}
function clearProgress(): void {
if (existsSync(getProgressFile())) writeFileSync(getProgressFile(), '', 'utf-8');
}
function ensureDir(path: string): void {
const d = dirname(path);
if (!existsSync(d)) mkdirSync(d, { recursive: true });
}
// ---------------------------------------------------------------------------
// CLI entry point
// ---------------------------------------------------------------------------
export { INTEGRITY_SUBCOMMANDS as SUBCOMMANDS } from '../cli/subcommands.ts';
export async function runIntegrity(args: string[]): Promise<void> {
const sub = args[0] as (typeof INTEGRITY_SUBCOMMANDS)[number] | undefined;
if (!sub || subcommandHelpRequested(args, ROUTERS.integrity)) {
printUsage();
return;
}
if (sub === 'check') {
await cmdCheck(args.slice(1));
return;
}
if (sub === 'auto') {
await cmdAuto(args.slice(1));
return;
}
if (sub === 'review') {
cmdReview();
return;
}
if (sub === 'reset-progress') {
clearProgress();
console.log('Cleared progress log:', getProgressFile());
return;
}
console.error(`Unknown subcommand: ${sub}`);
printUsage();
process.exit(1);
}
// ---------------------------------------------------------------------------
// check — read-only scan
// ---------------------------------------------------------------------------
async function cmdCheck(args: string[]): Promise<void> {
const jsonMode = args.includes('--json');
const limit = extractIntFlag(args, '--limit') ?? Infinity;
const typeFilter = extractFlag(args, '--type');
const engine = await connect();
try {
const res = await scanIntegrity(engine, { limit, typeFilter });
if (jsonMode) {
console.log(JSON.stringify({
pagesScanned: res.pagesScanned,
bareTweetHits: res.bareHits,
externalLinkCount: res.externalHits.length,
...(res.skipped.length ? { skipped: res.skipped } : {}),
}, null, 2));
return;
}
console.log(`Scanned ${res.pagesScanned} pages.`);
console.log(`Bare-tweet phrases: ${res.bareHits.length}`);
console.log(`External links (for optional dead-link check): ${res.externalHits.length}`);
if (res.skipped.length > 0) {
console.log(`Pages not scanned: ${res.skipped.length}`);
for (const skipped of res.skipped) console.log(` ${skipped.source_id}:${skipped.slug} (${skipped.reason}): ${skipped.message}`);
}
if (res.topPages.length > 0) {
console.log('\nTop 10 pages with bare-tweet references:');
for (const { slug, count } of res.topPages) {
console.log(` ${slug}: ${count} hit${count === 1 ? '' : 's'}`);
}
}
} finally {
await engine.disconnect();
}
}
// ---------------------------------------------------------------------------
// scanIntegrity — pure library function, callable from doctor
// ---------------------------------------------------------------------------
export interface IntegrityScanOptions {
/** Max pages to scan. Default Infinity. Doctor passes a sample limit (~500). */
limit?: number;
/** Slug prefix filter (e.g. "people") — matches slugs starting with `${typeFilter}/`. */
typeFilter?: string;
/**
* When true (default), batch-load pages via a single SQL query instead of
* sequential getPage() calls. Falls back to sequential on error (e.g. PGLite).
* Eliminates 500 round-trips through PgBouncer that caused doctor timeouts.
*/
batchLoad?: boolean;
}
export interface IntegrityScanResult {
pagesScanned: number;
bareHits: BareTweetHit[];
externalHits: ExternalLinkHit[];
/** Top 10 pages sorted by bare-tweet hit count, descending. */
topPages: Array<{ slug: string; count: number }>;
/** Pages the scan could not read (a per-page refusal); counted in pagesScanned, no hits. */
skipped: IntegritySkippedPage[];
}
/**
* Read-only integrity scan over the engine's pages. No network, no writes,
* no resolver calls. Called by `gbrain integrity check` for the full report
* and by `gbrain doctor` (non-fast) for a sampled health signal.
*
* Caller owns the engine lifecycle.
*/
export async function scanIntegrity(
engine: BrainEngine,
opts: IntegrityScanOptions = {},
): Promise<IntegrityScanResult> {
const { limit = Infinity, typeFilter, batchLoad = true } = opts;
// Fast path: single SQL query instead of N sequential getPage() calls.
// Eliminates ~500 round-trips through PgBouncer that caused doctor to
// timeout on transaction-mode pooling. Postgres-only: PGLite runs in
// process, where the sequential walk costs no network round-trips.
if (batchLoad && limit !== Infinity && engine.kind === 'postgres') {
try {
return await scanIntegrityBatch(engine, limit, typeFilter);
} catch (err) {
// GBRAIN_DEBUG=1 surfaces real Postgres errors (deadlock, connection
// drop, SQL bug) that would otherwise vanish into the sequential
// fallback. Quiet by default since the fallback is harmless.
if (process.env.GBRAIN_DEBUG) {
console.error(
'[integrity] batch path failed, falling back to sequential:',
err instanceof Error ? err.message : err,
);
}
}
}
// v0.32.8: listAllPageRefs replaces getAllSlugs+getPage N+1 pattern that
// silently defaulted to source_id='default' for non-default-source pages.
// Now we enumerate (slug, source_id) pairs and thread sourceId to getPage.
const allRefs = (await engine.listAllPageRefs()).sort((a, b) =>
a.slug.localeCompare(b.slug) || a.source_id.localeCompare(b.source_id)
);
const bareHits: BareTweetHit[] = [];
const externalHits: ExternalLinkHit[] = [];
let pagesScanned = 0;
const skipped: IntegritySkippedPage[] = [];
for (const { slug, source_id } of allRefs) {
if (typeFilter && !slug.startsWith(`${typeFilter}/`)) continue;
if (pagesScanned >= limit) break;
const page = await engine.getPage(slug, { sourceId: source_id });
if (!page) continue;
// Skip grandfathered pages (opted out of brain-integrity enforcement)
if ((page.frontmatter as Record<string, unknown> | undefined)?.validate === false) continue;
pagesScanned++;
try {
const hits = await integrityHitsOf(engine, page, source_id);
bareHits.push(...hits.bare);
externalHits.push(...hits.external);
} catch (error) {
skipped.push(skippedPage(error, { slug, source_id }));
}
}
const byPage = new Map<string, number>();
for (const h of bareHits) byPage.set(h.slug, (byPage.get(h.slug) ?? 0) + 1);
const topPages = [...byPage.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([slug, count]) => ({ slug, count }));
return { pagesScanned, bareHits, externalHits, topPages, skipped };
}
/**
* Batch-load integrity scan: fetches all candidate pages in a single SQL
* query, then scans in-memory. Reduces PgBouncer round-trips from ~500 to 1.
* A segmented page's body is read block by block from the same engine.
*/
async function scanIntegrityBatch(
engine: BrainEngine,
limit: number,
typeFilter?: string,
): Promise<IntegrityScanResult> {
const params: unknown[] = [limit];
const typeCondition = typeFilter ? `AND slug LIKE $${params.push(`${typeFilter}/%`)}` : '';
// Boolean validate is the documented contract; stringly-typed 'false' (quoted
// YAML) diverges from the sequential path's strict === false check. Intentional
// — gbrain lint should reject stringly-typed validate at write time.
const validateCondition = `AND (frontmatter->>'validate' IS NULL OR frontmatter->>'validate' != 'false')`;
// v0.32.8: scan ONE row per (source_id, slug) pair, not one per slug.
// Pre-fix used DISTINCT ON (slug) which collapsed multi-source rows into
// one — that was the bug class. Now batch parity matches the sequential
// listAllPageRefs() walk: integrity violations in non-default-source pages
// get reported instead of silently shadowed by their default-source twin.
// deleted_at IS NULL: the sequential walk (listAllPageRefs + getPage) never
// sees a soft-deleted page, so the batch must not either — otherwise the
// sampled doctor check reports hits from tombstones and, once they
// outnumber `limit`, stops reaching live pages.
const rows = await engine.executeRaw<Record<string, unknown>>(`
SELECT id, source_id, slug, compiled_truth, body_storage
FROM pages
WHERE deleted_at IS NULL ${typeCondition} ${validateCondition}
ORDER BY source_id, slug
LIMIT $1
`, params);
const bareHits: BareTweetHit[] = [];
const externalHits: ExternalLinkHit[] = [];
const skipped: IntegritySkippedPage[] = [];
for (const row of rows) {
const page = { id: Number(row.id), source_id: row.source_id as string, slug: row.slug as string,
compiled_truth: row.compiled_truth as string | null, body_storage: row.body_storage as BodyStorage };
try {
const hits = await integrityHitsOf(engine, page, page.source_id);
bareHits.push(...hits.bare);
externalHits.push(...hits.external);
} catch (error) {
skipped.push(skippedPage(error, page));
}
}
const byPage = new Map<string, number>();
for (const h of bareHits) byPage.set(h.slug, (byPage.get(h.slug) ?? 0) + 1);
const topPages = [...byPage.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([slug, count]) => ({ slug, count }));
return { pagesScanned: rows.length, bareHits, externalHits, topPages, skipped };
}
// ---------------------------------------------------------------------------
// auto — three-bucket repair
// ---------------------------------------------------------------------------
async function cmdAuto(args: string[]): Promise<void> {
const dryRun = args.includes('--dry-run');
const confidenceThreshold = extractFloatFlag(args, '--confidence') ?? 0.8;
const reviewLower = extractFloatFlag(args, '--review-lower') ?? 0.5;
const limit = extractIntFlag(args, '--limit') ?? Infinity;
const skipTweet = args.includes('--skip-bare-tweet');
const skipUrls = args.includes('--skip-urls');
const resume = !args.includes('--fresh');
if (confidenceThreshold < reviewLower) {
console.error('--confidence must be >= --review-lower');
process.exit(1);
}
ensureDir(gbrainPath());
const engine = await connect();
const registry = getDefaultRegistry();
registerBuiltinResolvers(registry);
// One writer PER SOURCE: BrainWriter scopes every read/write (and
// addTimelineEntry) to its sourceId — a single default-scoped writer used
// for every source's pages was the unscoped-check/scoped-write bug class.
const writersBySource = new Map<string, BrainWriter>();
const writerFor = (sourceId: string): BrainWriter => {
let w = writersBySource.get(sourceId);
if (!w) {
w = new BrainWriter(engine, { strictMode: 'off', sourceId });
writersBySource.set(sourceId, w);
}
return w;
};
const ctx: ResolverContext = {
engine,
config: {},
logger: {
info: (msg) => console.log(msg),
warn: (msg) => console.warn(msg),
error: (msg) => console.error(msg),
},
requestId: `integrity-auto-${Date.now()}`,
remote: false,
};
const seen = resume ? loadProgress() : (clearProgress(), new Set<string>());
let bucketAuto = 0;
let bucketReview = 0;
let bucketSkip = 0;
let bucketErr = 0;
let bucketDeadLink = 0;
let pagesProcessed = 0;
const skippedPages: IntegritySkippedPage[] = [];
const { createProgress } = await import('../core/progress.ts');
const { getCliOptions, cliOptsToProgressOptions } = await import('../core/cli-options.ts');
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
try {
// v0.32.8: listAllPageRefs enumerates (slug, source_id) pairs so we
// can thread sourceId to getPage. Pre-fix this defaulted to 'default'
// and silently skipped non-default-source pages.
const allRefs = (await engine.listAllPageRefs()).sort((a, b) =>
a.slug.localeCompare(b.slug) || a.source_id.localeCompare(b.source_id)
);
const toScan = allRefs.filter(r => !seen.has(progressKey(r.source_id, r.slug)));
progress.start('integrity.auto', toScan.length);
for (const { slug, source_id } of allRefs) {
if (pagesProcessed >= limit) break;
if (seen.has(progressKey(source_id, slug))) continue;
const writer = writerFor(source_id);
const page = await engine.getPage(slug, { sourceId: source_id });
if (!page) continue;
pagesProcessed++;
progress.tick(1, slug);
let found: Awaited<ReturnType<typeof integrityHitsOf>>;
try {
found = await integrityHitsOf(engine, page, source_id);
} catch (error) {
skippedPages.push(skippedPage(error, { slug, source_id }));
continue;
}
// Bare-tweet handling
if (!skipTweet) {
const hits = found.bare;
const handle = extractXHandleFromFrontmatter(page.frontmatter);
if (hits.length > 0 && handle) {
for (const hit of hits) {
try {
const result = await registry.resolve<{ handle: string; keywords: string }, {
url?: string; tweet_id?: string; text?: string; created_at?: string;
candidates: Array<{ tweet_id: string; text: string; created_at: string; score: number; url: string }>;
}>(
'x_handle_to_tweet',
{ handle, keywords: hit.rawLine.slice(0, 150) },
ctx,
);
if (result.confidence >= confidenceThreshold && result.value.url && result.value.tweet_id && result.value.created_at) {
await repairBareTweet({
writer, slug, hit, result, handle, dryRun,
});
bucketAuto++;
// Dry-run must NOT persist 'repaired' — the follow-on real
// run needs to revisit these slugs and actually write.
if (!dryRun) {
appendProgress({ slug, source_id, status: 'repaired', timestamp: new Date().toISOString() });
}
} else if (result.confidence >= reviewLower) {
appendReview({ slug, hit, result, handle });
bucketReview++;
if (!dryRun) {
appendProgress({ slug, source_id, status: 'reviewed', timestamp: new Date().toISOString() });
}
} else {
logSkip({ slug, hit, reason: `confidence ${result.confidence.toFixed(2)} below threshold ${reviewLower}` });
bucketSkip++;
if (!dryRun) {
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
}
}
} catch (e) {
bucketErr++;
logSkip({ slug, hit, reason: `resolver error: ${e instanceof Error ? e.message : String(e)}` });
if (!dryRun) {
appendProgress({ slug, source_id, status: 'error', timestamp: new Date().toISOString() });
}
}
}
} else if (hits.length > 0 && !handle) {
// Can't repair without a handle; log once per page
for (const hit of hits) {
logSkip({ slug, hit, reason: 'no x_handle in frontmatter to search from' });
}
bucketSkip += hits.length;
if (!dryRun) {
appendProgress({ slug, source_id, status: 'skipped', timestamp: new Date().toISOString() });
}
}
}
// Dead-link handling (no auto-repair; just surface)
if (!skipUrls) {
const externalHits = found.external;
// Limit to first few per page to keep the default run fast; --check
// gives the full picture.
for (const hit of externalHits.slice(0, 3)) {
try {
const result = await registry.resolve<
{ url: string },
{ reachable: boolean; status?: number; reason?: string }
>('url_reachable', { url: hit.url }, ctx);
if (!result.value.reachable) {
logSkip({
slug,
hit: { slug, line: hit.line, rawLine: hit.url, phrase: 'dead-link' },
reason: `dead link: ${result.value.reason ?? 'unknown'}`,
});
// Dead links have no confidence score and are never written to
// the review file (appendReview() is only called from the
// bare-tweet path above) — they land in the skip log via
// logSkip() a few lines up. Count them separately so the
// printed "Review queue" total only ever reflects what's
// actually in ~/.gbrain/integrity-review.md.
bucketDeadLink++;
}
} catch {
/* transient; don't fail the run */
}
}
}
}
progress.finish();
// Summary
console.log('');
console.log(`=== integrity auto summary${dryRun ? ' (DRY RUN)' : ''} ===`);
console.log(`Pages processed: ${pagesProcessed}`);
console.log(`Auto-repaired (≥${confidenceThreshold}): ${bucketAuto}`);
console.log(`Review queue (≥${reviewLower} <${confidenceThreshold}): ${bucketReview}`);
console.log(`Skipped (<${reviewLower}): ${bucketSkip}`);
if (bucketErr > 0) console.log(`Resolver errors: ${bucketErr}`);
if (bucketDeadLink > 0) console.log(`Dead links surfaced (see skipped log): ${bucketDeadLink}`);
if (skippedPages.length > 0) {
console.log(`Pages not scanned: ${skippedPages.length}`);
for (const skipped of skippedPages) console.log(` ${skipped.source_id}:${skipped.slug} (${skipped.reason}): ${skipped.message}`);
}
console.log(`\nReview queue: ${getReviewFile()}`);
console.log(`Skipped log: ${getLogFile()}`);
console.log(`Progress: ${getProgressFile()}`);
} finally {
await engine.disconnect();
}
}
// ---------------------------------------------------------------------------
// review — print the review queue location + count
// ---------------------------------------------------------------------------
function cmdReview(): void {
if (!existsSync(getReviewFile())) {
console.log(`No review queue yet. Run: gbrain integrity auto --confidence 0.8`);
return;
}
const content = readFileSync(getReviewFile(), 'utf-8');
const count = (content.match(/^## /gm) ?? []).length;
console.log(`Review queue: ${getReviewFile()}`);
console.log(`Entries: ${count}`);
console.log(`\nOpen with: $EDITOR ${getReviewFile()}`);
}
// ---------------------------------------------------------------------------
// Repair primitives
// ---------------------------------------------------------------------------
interface RepairArgs {
writer: BrainWriter;
slug: string;
hit: BareTweetHit;
result: ResolverResult<{ url?: string; tweet_id?: string; created_at?: string }>;
handle: string;
dryRun: boolean;
}
async function repairBareTweet(args: RepairArgs): Promise<void> {
const { writer, slug, hit, result, handle, dryRun } = args;
const tweetId = result.value.tweet_id!;
const createdAt = result.value.created_at!;
const dateISO = createdAt.slice(0, 10);
// Build the citation using Scaffolder (deterministic URL from API).
const cite = tweetCitation({ handle, tweetId, dateISO });
if (dryRun) {
console.log(`[dry-run] ${slug}:${hit.line} would append ${cite}`);
return;
}
// Read current, append citation to the flagged line, write back through
// BrainWriter so the transaction is atomic and the writer's grandfather
// opt-out can be cleared if validators pass post-repair.
const current = await (args.writer as unknown as { engine: BrainEngine })['engine']?.getPage?.(slug);
// fall back: use a direct engine handle via writer's internal ref is ugly;
// instead, use writer.transaction and read/write inside
await writer.transaction(async (tx) => {
// We can't read inside a transaction without engine access; set-wise,
// we fetch via the outer engine reference captured on the writer.
// Simpler: perform a read outside via setCompiledTruth which already
// handles "page not found" + merges with existing content server-side.
// However BrainWriter.setCompiledTruth requires the new body — we need
// to read first. Do the read here via the engine on the tx's context
// (the tx uses the same engine instance).
//
// Workaround: use setFrontmatterField + appendTimeline pattern. We
// leave the bare phrase alone and append a timeline entry with the
// citation. That's honest — we're adding evidence, not rewriting
// prose. Pages with `validate: false` in frontmatter stay flagged
// until a more thorough repair pass removes the bare phrase.
await tx.appendTimeline(slug, {
date: dateISO,
source: 'gbrain integrity --auto',
summary: `Bare-tweet reference repaired (line ${hit.line}): "${truncate(hit.rawLine, 80)}"`,
detail: cite,
});
}, {
config: {}, logger: { info: () => {}, warn: () => {}, error: () => {} },
requestId: 'integrity-repair', remote: false,
});
console.log(`repaired ${slug}:${hit.line} → ${cite}`);
// Silence unused var from earlier refactor
void current;
}
// ---------------------------------------------------------------------------
// Review queue + skip log
// ---------------------------------------------------------------------------
interface ReviewArgs {
slug: string;
hit: BareTweetHit;
result: ResolverResult<{
url?: string;
candidates: Array<{ tweet_id: string; text: string; created_at: string; score: number; url: string }>;
}>;
handle: string;
}
function appendReview(args: ReviewArgs): void {
ensureDir(getReviewFile());
const { slug, hit, result, handle } = args;
const block = [
`## ${slug}:${hit.line} (confidence ${result.confidence.toFixed(2)})`,
``,
`Handle: @${handle}`,
`Phrase: \`${hit.rawLine}\``,
``,
`Candidates:`,
...result.value.candidates.slice(0, 5).map((c, i) => ` ${i + 1}. ${c.url} — "${truncate(c.text, 80)}" (score ${c.score.toFixed(2)})`),
``,
'---',
'',
].join('\n');
appendFileSync(getReviewFile(), block, 'utf-8');
}
interface SkipArgs { slug: string; hit: BareTweetHit; reason: string }
function logSkip(args: SkipArgs): void {
ensureDir(getLogFile());
const entry = {
timestamp: new Date().toISOString(),
slug: args.slug,
line: args.hit.line,
phrase: args.hit.phrase,
raw: args.hit.rawLine.slice(0, 200),
reason: args.reason,
};
appendFileSync(getLogFile(), JSON.stringify(entry) + '\n', 'utf-8');
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
export function extractXHandleFromFrontmatter(fm: Record<string, unknown> | undefined): string | null {
if (!fm) return null;
for (const k of HANDLE_FRONTMATTER_KEYS) {
const v = fm[k];
if (typeof v === 'string' && v.trim().length > 0) {
return v.trim().replace(/^@/, '');
}
}
return null;
}
async function connect(): Promise<BrainEngine> {
const config = loadConfig();
if (!config) {
console.error('No brain configured. Run: gbrain init');
process.exit(1);
}
const engine = await createEngine(toEngineConfig(config));
await engine.connect(toEngineConfig(config));
return engine;
}
function extractFlag(args: string[], flag: string): string | undefined {
const idx = args.findIndex(a => a === flag || a.startsWith(`${flag}=`));
if (idx === -1) return undefined;
const arg = args[idx];
if (arg.includes('=')) return arg.slice(arg.indexOf('=') + 1);
return args[idx + 1];
}
function extractIntFlag(args: string[], flag: string): number | undefined {
const v = extractFlag(args, flag);
if (v === undefined) return undefined;
const n = parseInt(v, 10);
return Number.isFinite(n) ? n : undefined;
}
function extractFloatFlag(args: string[], flag: string): number | undefined {
const v = extractFlag(args, flag);
if (v === undefined) return undefined;
const n = parseFloat(v);
return Number.isFinite(n) ? n : undefined;
}
function truncate(s: string, n: number): string {
return s.length <= n ? s : s.slice(0, n - 3) + '...';
}
export function printUsage(): void {
console.log(`Usage: gbrain integrity <subcommand> [options]
Subcommands:
check Read-only report (pages scanned, bare tweets found)
check --type people Scope to people/ pages
check --limit N --json JSON output for N pages
auto [options] Three-bucket repair loop
--confidence 0.8 Auto-repair threshold (default 0.8)
--review-lower 0.5 Review-queue lower bound (default 0.5)
--dry-run Report what would change, no writes
--limit N Process at most N pages (resumable)
--fresh Ignore progress file; start over
--skip-bare-tweet Skip bare-tweet detection
--skip-urls Skip dead-link detection
review Print review-queue path + entry count
reset-progress Clear ~/.gbrain/integrity-progress.jsonl
Paths:
Review queue: ~/.gbrain/integrity-review.md
Skip log: ~/.gbrain/integrity.log.jsonl
Progress: ~/.gbrain/integrity-progress.jsonl
`);
}