Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
47.9 KB
Raw Blame
import { readSourceFileSync, writeSourceFileSync, hasSourceFilesystemLock, withSourceFilesystemLock, assertSourceFilesystemActive } from '../core/minions/source-filesystem.ts';
/**
 * gbrain lint — Deterministic brain page quality checker.
 *
 * Zero LLM calls. Catches common quality issues:
 * - LLM preamble artifacts ("Of course! Here is...")
 * - Placeholder dates (YYYY-MM-DD, XX-XX left unfilled)
 * - Missing required frontmatter fields
 * - Broken citations (unclosed brackets, missing dates)
 * - Empty/stub sections
 * - Wrapping code fences from LLM output
 * - Page types the active schema pack does not declare (type-undeclared)
 *
 * Usage:
 *   gbrain lint <dir>              # report issues
 *   gbrain lint <dir> --fix        # auto-fix what's fixable
 *   gbrain lint <dir> --fix --dry-run  # preview fixes
 *   gbrain lint <file.md>          # lint single file
 */

import { readdirSync, statSync, lstatSync, existsSync } from 'fs';
import { join, relative, dirname } from 'path';
import { isAborted } from '../core/abort-check.ts';
import { parseMarkdown, type ParseValidationCode } from '../core/markdown.ts';
import { parseDateLoose } from '../core/effective-date.ts';
import {
  assessContentSanity,
  type OperatorLiteral,
  DEFAULT_BYTES_WARN,
} from '../core/content-sanity.ts';
import { loadOperatorLiterals } from '../core/content-sanity-literals.ts';
import { loadConfig, loadConfigWithEngine, gbrainPath } from '../core/config.ts';
import { isDurabilityHardened, commitWriteThroughFile } from '../core/brain-repo-durability.ts';
import type { BrainEngine } from '../core/engine.ts';
import { loadActivePack } from '../core/schema-pack/load-active.ts';
import { loadActivePackForLocalEngine } from '../core/schema-pack/best-effort.ts';
import { safeCliToken, sanitizeTypeForDisplay, storedTypeMissesPack, type TypeUsagePack } from '../core/schema-pack/type-usage.ts';
import { parseLineGrammar } from '../core/line-grammar.ts';
import { stripCodeBlocks } from '../core/markdown-code.ts';
import { pathToSlug } from '../core/sync.ts';
import { isManagedBrain } from '../core/cycle/phase-table.ts';
import { maintenancePreflight, publishMaintenancePage, type MaintenanceAuthority } from '../core/persistence/prepared-maintenance.ts';
import { fileMatchesSnapshot, prepareFileTarget } from '../core/persistence/page-prepare.ts';
import { scannerSlugRootMode, scannerSourcePath } from '../core/write-through.ts';
import { heldFileDiagnostic, writeFailureDiagnostic } from '../core/persistence/verb-errors.ts';
import { loadActivePackForEngine } from '../core/schema-pack/engine-resolution.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import { readFix } from '../core/ops/op-fix.ts';
import { ERROR_CATALOGUE } from '../core/error-catalogue.ts';
import { cliRenderContext, docsUrl, renderAction, toAgentError, type RenderedAction } from '../core/agent-output.ts';
import type { ParseOpts } from '../core/markdown.ts';

export interface LintIssue {
  file: string;
  line: number;
  rule: string;
  message: string;
  fixable: boolean;
  /** The stable machine code: a repair that was not applied (`managed_write_pending` from a managed brain's coordinator, `fix_not_writable` when the file refused the write), or a listed file removed before lint read it (`file_removed_during_scan`). */
  code?: 'managed_write_pending' | 'fix_not_writable' | 'file_removed_during_scan';
  /** Why: the coordinator refusal's reason (`file_database_drift`, `held_file`, `canonical_file_missing`, ...), `revision_changed`, `not_indexed`, or the lowercased errno (`eacces`, `eperm`, `erofs`). */
  reason?: string;
  /** The next step, rendered for the CLI: the coordinator refusal's own fix, or the owner's permission fix. */
  fix?: RenderedAction;
  docs?: string;
}

/** Map of frontmatter validation codes to lint rule names. Stable across
 *  releases — agents and CI consumers can target specific rule names. */
const FRONTMATTER_RULE_NAMES: Record<ParseValidationCode, string> = {
  MISSING_OPEN: 'frontmatter-missing-open',
  MISSING_CLOSE: 'frontmatter-missing-close',
  YAML_PARSE: 'frontmatter-yaml-parse',
  SLUG_MISMATCH: 'frontmatter-slug-mismatch',
  NULL_BYTES: 'frontmatter-null-bytes',
  NESTED_QUOTES: 'frontmatter-nested-quotes',
  NON_STRING_FIELD: 'frontmatter-non-string-field',
  EMPTY_FRONTMATTER: 'frontmatter-empty',
};

/** Codes whose lint findings are fixable by `gbrain frontmatter validate --fix`. */
const FRONTMATTER_FIXABLE: ReadonlySet<ParseValidationCode> = new Set<ParseValidationCode>([
  'MISSING_CLOSE',
  'NULL_BYTES',
  'NESTED_QUOTES',
]);

// ── LLM artifact patterns ──────────────────────────────────────────

// Anchored to the start of the page body (no `g`/`m` flags): an LLM prepends
// a preamble to its output, so a line further down that merely begins
// "Sure! Here is..." (a quoted reply, a transcript turn) is real content.
const LLM_PREAMBLES = [
  /^Of course\.?\s*Here is (?:a |the )?(?:detailed |comprehensive |updated )?(?:brain )?page[^.\n]*\.?\s*\n*/i,
  /^Certainly\.?\s*Here is[^.\n]*\.?\s*\n*/i,
  /^Here is (?:a |the )?(?:detailed |comprehensive |updated )?(?:brain )?page[^.\n]*\.?\s*\n*/i,
  /^I've (?:created|updated|written|prepared) (?:a |the )?(?:detailed |comprehensive )?(?:brain )?page[^.\n]*\.?\s*\n*/i,
  /^Sure(?:!|,)?\s*Here (?:is|are)[^.\n]*\.?\s*\n*/i,
  /^Absolutely\.?\s*Here[^.\n]*\.?\s*\n*/i,
];

const LEADING_FRONTMATTER = /^---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/;
const LEADING_MARKDOWN_FENCE = /^```(?:markdown|md)[ \t]*\r?\n/;

/**
 * The leading LLM preamble run (stacked preamble lines) at a body start: the
 * top of the file, right after a leading frontmatter block, or right after a
 * whole-page ```markdown wrapper, leading blank lines skipped. Returns the
 * [start, end) span to remove, or null. Detection and repair both use it.
 */
function findLeadingPreamble(content: string): { start: number; end: number } | null {
  const skipBlank = (i: number) => i + (/^\s*/.exec(content.slice(i))?.[0].length ?? 0);
  const anchors = [0, LEADING_FRONTMATTER.exec(content)?.[0].length, LEADING_MARKDOWN_FENCE.exec(content)?.[0].length]
    .filter((a): a is number => a !== undefined).map(skipBlank);
  for (const start of anchors) {
    let end = start;
    for (let guard = 0; guard < 16; guard++) {
      const hit = LLM_PREAMBLES.map(p => p.exec(content.slice(end))).find(m => m && m[0].length > 0);
      if (!hit) break;
      end += hit[0].length;
    }
    if (end > start) return { start, end };
  }
  return null;
}

// ── Rules ──────────────────────────────────────────────────────────

/**
 * #5433: frontmatter keys that count as temporal provenance for the
 * `missing-created` rule. `created` first for the legacy name; the rest are
 * what effective-date resolution and the native serializer actually write.
 */
const TEMPORAL_KEYS = ['created', 'event_date', 'date', 'published', 'captured_at', 'ingested_at'] as const;

/**
 * Per-call options for `lintContent`. Tests pass content-sanity opts
 * directly so the linter can be exercised without an engine.
 * Production callers (`runLintCore`) resolve effective config first
 * via the file/env/DB precedence chain and pass through.
 */
export interface LintContentOpts {
  /** v0.41 content-sanity thresholds + operator literals. When omitted,
   *  the assessor uses its built-in defaults (50K warn, 500K block,
   *  built-in junk patterns only). */
  contentSanity?: {
    bytes_warn?: number;
    bytes_block?: number;
    junk_patterns_enabled?: boolean;
    disabled?: boolean;
    max_markup_ratio?: number;
    prose_check_enabled?: boolean;
    operator_literals?: ReadonlyArray<OperatorLiteral>;
    /** #4702: built-in junk-pattern names to skip (see content-sanity.ts). */
    disabled_patterns?: string[];
  };
  /** #5626: active schema pack vocabulary for the `type-undeclared` rule.
   *  Null/omitted (no pack resolved) skips the rule. */
  typePack?: LintTypePack | null;
}

export type LintTypePack = TypeUsagePack & { name: string };

export function lintContent(content: string, filePath: string, opts: LintContentOpts = {}): LintIssue[] {
  const issues: LintIssue[] = [];
  const lines = content.split('\n');

  // ── Frontmatter validation (delegates to parseMarkdown(validate:true)) ──
  // This is the single source of truth for frontmatter shape rules. Each
  // ParseValidationCode maps to a stable lint rule name in
  // FRONTMATTER_RULE_NAMES. Keeps brain-page lint, doctor's
  // frontmatter_integrity subcheck, and the frontmatter CLI in lockstep.
  const parsed = parseMarkdown(content, filePath, { validate: true });
  for (const err of parsed.errors ?? []) {
    // Skip MISSING_OPEN — the legacy `no-frontmatter` rule below covers this
    // exact case with a stable rule name. Emitting both is double-reporting.
    if (err.code === 'MISSING_OPEN') continue;
    issues.push({
      file: filePath,
      line: err.line ?? 1,
      rule: FRONTMATTER_RULE_NAMES[err.code],
      message: err.message,
      fixable: FRONTMATTER_FIXABLE.has(err.code),
    });
  }

  // Rule: line-grammar near-misses (a relation or fact line that will not be
  // read as written). Read-only; the fix is in each message. Syntax-only: a
  // file check reads no brain settings or schema pack, so it never implies an
  // edge will be stored; `gbrain get <slug> --grammar-diagnostics` is the
  // effective check against the brain.
  for (const d of parseLineGrammar(content).diagnostics) {
    issues.push({ file: filePath, line: d.line, rule: 'line-grammar',
      message: `${d.message} (${d.reason}; syntax-only check: run \`gbrain get <slug> --grammar-diagnostics\` for what the brain reads)`, fixable: false });
  }

  // Rule: LLM preamble artifacts (only a leading run; see findLeadingPreamble)
  const preamble = findLeadingPreamble(content);
  if (preamble) {
    issues.push({
      file: filePath, line: content.slice(0, preamble.start).split('\n').length, rule: 'llm-preamble',
      message: 'LLM preamble artifact detected (e.g., "Of course! Here is...")',
      fixable: true,
    });
  }

  // Rule: Wrapping code fences (```markdown ... ```)
  // Detector intentionally has NO /m flag so ^/$ match start/end of the whole
  // file, not inner lines. Keeps detector in sync with fixContent() below,
  // which also has no /m flag. Without this, lint reports "fixable" false
  // positives on any page that simply contains a ```markdown code block, but
  // fixContent can never strip them (its regex only matches whole-file wrappers).
  if (content.match(/^```(?:markdown|md)\s*\n/) && content.match(/\n```\s*$/)) {
    issues.push({
      file: filePath, line: 1, rule: 'code-fence-wrap',
      message: 'Page wrapped in ```markdown code fences (LLM artifact)',
      fixable: true,
    });
  }

  // Rule: Placeholder dates. #3958/#6133: code is not a placeholder — a page
  // DOCUMENTING date formats (```\ncreated: YYYY-MM-DD\n``` or `YYYY-MM-DD`)
  // is not a page with an unfilled one. stripCodeBlocks masks fenced blocks
  // and inline spans in place, so line numbers stay exact.
  // #6538: in the body, a YYYY-MM-DD token followed by `-word`, `.ext` or `/` is a filename/path template
  // (`YYYY-MM-DD-topic.md`), not an unfilled date; frontmatter stays strict.
  const prose = stripCodeBlocks(content).split('\n');
  const frontmatterLines = content.startsWith('---') ? lines.indexOf('---', 1) : -1;
  for (let i = 0; i < lines.length; i++) {
    const datePlaceholder = i <= frontmatterLines ? /\bYYYY-MM-DD\b/ : /\bYYYY-MM-DD\b(?![-./][A-Za-z0-9]|\/)/;
    if (prose[i].match(datePlaceholder) || prose[i].match(/\bXX-XX\b/) || prose[i].match(/\b\d{4}-XX-XX\b/)) {
      issues.push({
        file: filePath, line: i + 1, rule: 'placeholder-date',
        message: `Placeholder date found: ${lines[i].trim().slice(0, 60)}`,
        fixable: false,
      });
    }
  }

  // Rule: Missing frontmatter
  if (content.startsWith('---')) {
    const fmEnd = content.indexOf('---', 3);
    if (fmEnd > 0) {
      const fm = content.slice(3, fmEnd);
      if (!fm.match(/^title:/m)) {
        issues.push({
          file: filePath, line: 1, rule: 'missing-title',
          message: 'Frontmatter missing required field: title',
          fixable: false,
        });
      }
      if (!fm.match(/^type:/m)) {
        issues.push({
          file: filePath, line: 1, rule: 'missing-type',
          message: 'Frontmatter missing required field: type',
          fixable: false,
        });
      } else if (opts.typePack && parsed.typeExplicit && storedTypeMissesPack(parsed.type, opts.typePack)) {
        // #5626: same classification as the schema lint stored_type_undeclared rule.
        const t = sanitizeTypeForDisplay(parsed.type);
        issues.push({
          file: filePath,
          line: content.slice(0, 3 + fm.search(/^type:/m)).split('\n').length,
          rule: 'type-undeclared',
          message: `Page type '${t}' is not declared in active schema pack '${opts.typePack.name}' (not a page type or alias). ` +
            `Use a declared type, or declare it: gbrain schema add-type ${safeCliToken(parsed.type) ?? '<type>'} (with its primitive and prefix)`,
          fixable: false,
        });
      }
      // #5433: the native serializer never writes `created` and effective-date
      // resolution reads event_date / date / published, so any parseable
      // temporal key is provenance enough. The rule name stays for consumers;
      // it is never fixable (ingest time is not creation time, see #3958).
      // A block YAML could not read is already a frontmatter-yaml-parse finding.
      const yamlUnreadable = parsed.errors?.some(e => e.code === 'YAML_PARSE') ?? false;
      if (!yamlUnreadable && !TEMPORAL_KEYS.some(key => parseDateLoose(parsed.frontmatter[key]) !== null)) {
        issues.push({
          file: filePath, line: 1, rule: 'missing-created',
          message: `Frontmatter has no parseable date under any of: ${TEMPORAL_KEYS.join(', ')}`,
          fixable: false,
        });
      }
    }
  } else {
    // No frontmatter at all
    issues.push({
      file: filePath, line: 1, rule: 'no-frontmatter',
      message: 'Page has no YAML frontmatter',
      fixable: false,
    });
  }

  // Rule: Broken citations (unclosed [Source: ...)
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];
    // Open [Source: without closing ]
    if (line.match(/\[Source:[^\]]*$/) && !(i + 1 < lines.length && lines[i + 1].match(/^\s*[^\[]*\]/))) {
      issues.push({
        file: filePath, line: i + 1, rule: 'broken-citation',
        message: 'Unclosed [Source: ...] citation',
        fixable: false,
      });
    }
  }

  // Rule: Empty/stub sections. #6257: headings and section boundaries are
  // found in the code-masked text (a `## ` line inside a fence is not a
  // section); the body and title are read from the original at the same
  // offsets, so a section whose body is only a code block is not empty.
  const masked = prose.join('\n');
  const sectionPattern = /^##\s+(.+)$/gm;
  let sectionMatch;
  while ((sectionMatch = sectionPattern.exec(masked)) !== null) {
    const sectionStart = sectionMatch.index + sectionMatch[0].length;
    const nextSection = masked.indexOf('\n## ', sectionStart);
    const sectionBody = content.slice(sectionStart, nextSection > 0 ? nextSection : undefined).trim();

    if (sectionBody === '' || sectionBody === '[No data yet]' || sectionBody === '*[To be filled by agent]*') {
      const lineNum = content.slice(0, sectionMatch.index).split('\n').length;
      const title = content.slice(sectionStart - sectionMatch[1].length, sectionStart);
      issues.push({
        file: filePath, line: lineNum, rule: 'empty-section',
        message: `Empty section: ## ${title}`,
        fixable: false,
      });
    }
  }

  // v0.41 content-sanity rules. Two new lint rules (huge-page +
  // scraper-junk) backed by the shared assessor in
  // src/core/content-sanity.ts so the threshold + pattern set stays
  // in sync with the ingest gate at importFromContent. Kill-switch
  // (contentSanity.disabled) suppresses both.
  //
  // Bytes are measured against the parsed body (compiled_truth +
  // timeline) for parity with doctor's `oversized_pages` check (D2).
  // The earlier file-byte design disagreed with doctor on pages with
  // large frontmatter; pulling from parsed keeps the surfaces aligned
  // on the operationally-meaningful axis (embed pipeline input).
  const cs = opts.contentSanity ?? {};
  if (cs.disabled !== true) {
    const operator_literals = cs.junk_patterns_enabled !== false
      ? (cs.operator_literals ?? [])
      : [];
    const sanity = assessContentSanity({
      compiled_truth: parsed.compiled_truth,
      timeline: parsed.timeline ?? '',
      title: parsed.title,
      bytes_warn: cs.bytes_warn,
      bytes_block: cs.bytes_block,
      max_markup_ratio: cs.max_markup_ratio,
      prose_check_enabled: cs.prose_check_enabled,
      page_kind: parsed.type,
      extra_literals: operator_literals,
      // #4702: same resolution as import — lint and import disagreeing about
      // which patterns are live would make lint report a page as junk that
      // import is configured to let through.
      disabled_patterns: Array.isArray(cs.disabled_patterns)
        ? cs.disabled_patterns
        : undefined,
    });
    // Rule: huge-page fires for both oversize_warn (over warn threshold)
    // AND oversize_block (over block threshold). Operator sees the same
    // rule name in both cases; the message names the actual byte count.
    if (sanity.reasons.includes('oversize_warn') || sanity.reasons.includes('oversize_block')) {
      const threshold = sanity.reasons.includes('oversize_block') ? 'block' : 'warn';
      issues.push({
        file: filePath, line: 1, rule: 'huge-page',
        message: `Page body is ${sanity.bytes} bytes (exceeds ${threshold} threshold)`,
        fixable: false,
      });
    }
    // Rule: scraper-junk fires on any built-in pattern or operator literal hit.
    // Message names which pattern(s) matched so the brain-author can
    // either delete the file from their source repo or audit the scraper.
    if (sanity.junk_pattern_matches.length > 0 || sanity.literal_substring_matches.length > 0) {
      const matched = [
        ...sanity.junk_pattern_matches,
        ...sanity.literal_substring_matches,
      ].join(', ');
      issues.push({
        file: filePath, line: 1, rule: 'scraper-junk',
        message: `Matched junk pattern(s): ${matched}`,
        fixable: false,
      });
    }
    // Rule: markup-heavy fires when the fuzzy prose pass flags the page as
    // boilerplate-shaped (issue #1699). At ingest this FLAGS (page stays
    // searchable, agent warned) rather than hides — surfacing it in lint
    // lets a brain-author notice nav/boilerplate scrapes in their source.
    if (sanity.reasons.includes('high_markup')) {
      issues.push({
        file: filePath, line: 1, rule: 'markup-heavy',
        message: `Markup ratio ${sanity.markup_ratio?.toFixed(2)} exceeds threshold (looks like nav/boilerplate; flagged, not hidden)`,
        fixable: false,
      });
    }
  }

  return issues;
}

/**
 * @deprecated #5433: no longer part of `fixContent`. Ingest time is when a
 * file entered the brain, not when its content was created, and the cycle's
 * lint phase ran this on every native page, diverging the file from the DB
 * row that serialized it. Kept for callers who promote explicitly.
 *
 * Promotes `captured_at:` (preferred) or `ingested_at:` to `created:` when
 * the frontmatter has no `created:` of its own. The value is copied verbatim
 * (quoting preserved) and inserted directly below the source line. No-op
 * when there is no frontmatter, `created:` already exists, or neither
 * capture field is present.
 */
export function promoteCreatedFromCapture(content: string): string {
  if (!content.startsWith('---')) return content;
  const fmEnd = content.indexOf('---', 3);
  if (fmEnd <= 0) return content;
  const fm = content.slice(3, fmEnd);
  if (fm.match(/^created:/m)) return content;
  const src = fm.match(/^captured_at:[ \t]*(\S.*)$/m) ?? fm.match(/^ingested_at:[ \t]*(\S.*)$/m);
  if (!src || src.index === undefined) return content;
  const insertAt = 3 + src.index + src[0].length;
  return content.slice(0, insertAt) + `\ncreated: ${src[1].trim()}` + content.slice(insertAt);
}

/** Auto-fix fixable issues */
export function fixContent(content: string): string {
  let fixed = content;

  // Fix LLM preambles: only the leading run at a body start.
  const preamble = findLeadingPreamble(fixed);
  if (preamble) fixed = fixed.slice(0, preamble.start) + fixed.slice(preamble.end);

  // Fix wrapping code fences
  fixed = fixed.replace(/^```(?:markdown|md)\s*\n/, '');
  fixed = fixed.replace(/\n```\s*$/, '');

  // Clean up excessive blank lines left by fixes
  fixed = fixed.replace(/\n{3,}/g, '\n\n');

  return fixed.trim() + '\n';
}

/**
 * Resolve effective content-sanity opts for lint (D1: file/env first,
 * lift DB-plane when an engine is reachable).
 *
 * File/env path is sync via `loadConfig()`; DB-plane lift requires a
 * brief engine open. Best-effort: any engine failure (no brain
 * configured, connection refused, transient error) falls through to
 * the file/env values. CI without `~/.gbrain/` falls through
 * immediately since `loadConfig()` returns minimal config.
 *
 * Also loads the operator literals file (`~/.gbrain/junk-substrings.txt`)
 * once per lint invocation so multi-file lint runs amortize the read.
 */
async function resolveLintContentSanity(
  sharedEngine?: BrainEngine,
): Promise<{ contentSanity: LintContentOpts['contentSanity']; typePack: LintTypePack | null }> {
  const base = loadConfig();
  let cs = base?.content_sanity;
  // #5626: the active pack resolves from the same engine (DB-plane
  // schema_pack, tier 4) when one is reachable, else from file/env config.
  let typePack: LintTypePack | null | undefined;
  const packFromEngine = async (engine: BrainEngine) => {
    typePack = (await loadActivePackForLocalEngine(engine))?.manifest ?? null;
  };

  // DB-plane lift. issue #1678: when the caller already holds a live engine
  // (the cycle's lint phase, the Minion lint handler), REUSE it — do NOT
  // create + disconnect our own. A self-created engine here is module-style
  // (createEngine without poolSize wraps the db.ts singleton), so its
  // disconnect() cascades to db.disconnect() and NULLS the shared singleton
  // mid-cycle — which broke every subsequent cycle phase with a misleading
  // "connect() has not been called". Reusing the live engine reads the same
  // 4 config keys with zero connection churn.
  if (sharedEngine) {
    try {
      const lifted = await loadConfigWithEngine(sharedEngine, base);
      cs = lifted?.content_sanity ?? cs;
    } catch {
      // best-effort; fall through to file/env values.
    }
    await packFromEngine(sharedEngine);
  } else {
    // Standalone path (CLI `gbrain lint`, which is CLI_ONLY and shares no
    // engine): only attempt when the file/env config suggests an engine is
    // configured. Avoids spinning up a fresh PGLite just to read 4 config
    // keys in a CI lint run that has no brain at all. Safe to create +
    // disconnect here because nothing else shares this process's singleton.
    const hasEngineConfig = !!(base?.database_url || base?.database_path);
    if (hasEngineConfig) {
      try {
        const { createEngine } = await import('../core/engine-factory.ts');
        const engine = await createEngine({
          engine: base!.engine,
          database_url: base!.database_url,
          database_path: base!.database_path,
        });
        try {
          await engine.connect({});
          const lifted = await loadConfigWithEngine(engine, base);
          cs = lifted?.content_sanity ?? cs;
          await packFromEngine(engine);
        } finally {
          await engine.disconnect().catch(() => { /* best-effort cleanup */ });
        }
      } catch {
        // Engine unreachable or failed mid-probe — fall through to
        // file/env values. Lint should never block on engine state.
      }
    }
  }

  // Operator literals: always attempt to load (cheap FS read; missing
  // file is the common case and returns []). Skip when kill-switch
  // is on or junk patterns explicitly disabled to match the assessor's
  // own bypass logic exactly.
  const operator_literals = cs?.disabled === true || cs?.junk_patterns_enabled === false
    ? []
    : loadOperatorLiterals();

  if (typePack === undefined) {
    try {
      typePack = (await loadActivePack({ cfg: base, remote: false })).manifest;
    } catch {
      typePack = null;
    }
  }

  return {
    contentSanity: { ...cs, operator_literals },
    typePack,
  };
}

/**
 * Directories never containing knowledge pages, skipped by default.
 * Deliberately tiny: only vendored dependency trees qualify. Anything
 * more opinionated (README.md, CHANGELOG.md, test/) is repo policy —
 * callers opt in via `--exclude` / `LintOpts.exclude`. Dot- and
 * underscore-prefixed entries are already skipped by the walk.
 */
const DEFAULT_LINT_EXCLUDE_DIRS = new Set(['node_modules']);

/** Collect markdown files from a directory */
function collectPages(dir: string, extraExcludes: string[] = []): string[] {
  const extra = new Set(extraExcludes);
  const pages: string[] = [];
  function walk(d: string) {
    for (const entry of readdirSync(d)) {
      if (entry.startsWith('.') || entry.startsWith('_')) continue;
      const full = join(d, entry);
      if (lstatSync(full).isDirectory()) {
        if (DEFAULT_LINT_EXCLUDE_DIRS.has(entry) || extra.has(entry)) continue;
        walk(full);
      } else if (entry.endsWith('.md')) {
        if (extra.has(entry)) continue;
        pages.push(full);
      }
    }
  }
  walk(dir);
  return pages.sort();
}

export interface LintOpts {
  target: string;
  fix?: boolean;
  dryRun?: boolean;
  /** v0.41: optional pre-resolved content-sanity opts. When omitted,
   *  `runLintCore` resolves via the file/env/DB chain. Tests inject
   *  this directly to bypass the FS + engine layers. */
  contentSanity?: LintContentOpts['contentSanity'];
  /** #5626: optional pre-resolved pack vocabulary (null disables the
   *  `type-undeclared` rule). When omitted, resolved with content sanity. */
  typePack?: LintTypePack | null;
  /** issue #1678: a live, already-connected engine to REUSE for the
   *  content-sanity DB-plane config lift. Callers with a shared engine (the
   *  cycle lint phase, Minion lint handlers) MUST pass it so lint doesn't
   *  create + disconnect a competing module-style engine that nulls the
   *  shared db singleton mid-cycle. */
  engine?: BrainEngine;
  /**
   * #5180: source id for the coordinator write path. On a managed brain
   * (persistence enabled) fixes are published through the persistence
   * coordinator for this source instead of written to the worktree, which
   * the managed filesystem guard refuses. Defaults to 'default'.
   */
  sourceId?: string;
  /**
   * #1972: cooperative-abort signal. lint's per-page work is synchronous, so
   * without a periodic yield the event loop can't deliver an abort and a
   * very large lint would block past the worker's 30s force-evict. The loop
   * yields + checks this every 200 pages.
   */
  signal?: AbortSignal;
  /**
   * #2649: extra dir/file basenames to skip while collecting pages, in
   * addition to node_modules and dot/underscore entries. For mixed-content
   * repos (knowledge pages alongside software trees). Ignored for
   * single-file targets.
   */
  exclude?: string[];
  /**
   * W0 fix-wave (Tier-1 #14): per-page hook fired for every page WITH
   * issues, after this run's fix attempt for that page — fixedCount is the
   * number of fixes just applied (0 when --fix is off or nothing was
   * fixable). The CLI passes a printer so human detail and the aggregate
   * counts come from ONE scan — pre-fix, runLint ran its own full
   * read+lint+fix loop and THEN called runLintCore for the summary, linting
   * every page twice and reporting "0 auto-fixed" because the second pass
   * saw already-fixed files.
   */
  onPageIssues?: (relPath: string, issues: LintIssue[], fixedCount: number) => void;
  /** Companion to onPageIssues: per-page progress tick (CLI progress bar). */
  onPageScanned?: () => void;
  /**
   * Fired once with the collected page count before scanning starts, so the
   * CLI can size its progress bar without walking the tree a second time.
   */
  onPagesCollected?: (count: number) => void;
}

export interface LintResult {
  pages_scanned: number;
  pages_with_issues: number;
  total_issues: number;
  /** #3958: how many of total_issues are fixable — the CLI's "Run with
   *  --fix" hint only prints when this is non-zero, so an all-unfixable
   *  report can't send the operator on a no-op --fix run. */
  total_fixable: number;
  total_fixed: number;
  dryRun: boolean;
  applied_fix: boolean;
  /** #5180: where fixes went — the worktree (legacy), the persistence
   *  coordinator (managed brain), or nowhere (report-only / dry-run). */
  write_path: 'filesystem' | 'coordinator' | 'none';
  /** Fixes not applied this run: a managed brain could not publish them (#5180:
   *  no indexed page, or the page changed mid-scan; `managed-write-pending`),
   *  or the file refused the write (`fix-not-writable`). Each is a non-fixable
   *  issue and is retried on the next run. */
  fix_pending: number;
  /** The first MAX_PENDING_ISSUES of those issues with their code, reason and fix. */
  pending_issues: LintIssue[];
}

const MAX_PENDING_ISSUES = 50;
const MANAGED_LINT_DOCS = 'docs/guides/concurrent-writes.md#lint-repairs-waiting-on-a-managed-brain';

function pendingIssue(file: string, slug: string, reason: string, error: OperationError | null, message?: string): LintIssue {
  const envelope = error ? toAgentError(error, { transport: 'cli', command: 'lint', render: cliRenderContext() }) : null;
  return { file, line: 1, rule: 'managed-write-pending', fixable: false, code: 'managed_write_pending', reason,
    message: message ?? `fix not applied: ${slug}: ${envelope!.message}`,
    ...(envelope?.fix ? { fix: envelope.fix } : {}), docs: docsUrl(MANAGED_LINT_DOCS) };
}

/** A file the scan listed was gone by its scan read (deleted or renamed concurrently); the rest of the run continues. */
function removedDuringScanIssue(file: string, target: string): LintIssue {
  return { file, line: 1, rule: 'file-removed-during-scan', fixable: false, code: 'file_removed_during_scan',
    message: `${file} was removed after lint listed it and before lint read it, so it was not linted. Re-run lint to check the files that exist now.`,
    fix: renderAction({ argv: ['gbrain', 'lint', target], consent: [], actor: 'agent', requires_exclusive: false,
      why: 'Lists the tree again and lints the files present now.' }, cliRenderContext()),
    docs: docsUrl(ERROR_CATALOGUE.file_removed_during_scan.docs) };
}

const UNWRITABLE_CODES: ReadonlySet<string> = new Set(['EACCES', 'EPERM', 'EROFS']);

/**
 * Write one filesystem lint fix. A file the process may not write (EACCES,
 * EPERM, EROFS: permissions or a read-only mount) is left unchanged and
 * returned as a non-fixable `fix-not-writable` issue so the rest of the tree
 * is still linted; any other write error propagates.
 */
function writeLintFixOrRefusal(page: string, relPath: string, fixed: string): LintIssue | null {
  try {
    writeSourceFileSync(page, fixed);
    return null;
  } catch (e) {
    const errno = (e as NodeJS.ErrnoException | null)?.code;
    if (typeof errno !== 'string' || !UNWRITABLE_CODES.has(errno)) throw e;
    const envelope = toAgentError(opError('fix_not_writable', `fix not applied: ${relPath} is not writable (${errno}); the file was left unchanged.`,
      'Make the file writable by the user running gbrain, or pass its directory or file name to `gbrain lint --exclude` (for the cycle, add it to `cycle.lint_exclude` with `gbrain config set`), then lint again.', {
        why: 'Lint repairs files in place, and this file refused the write (its permissions or a read-only mount).',
        fix: { consent: [], actor: 'user', requires_exclusive: false, why: 'Only the file owner can change its permissions or mount.',
          user_message: `Make ${relPath} writable for gbrain, or exclude it from lint.` },
      }), { transport: 'cli', command: 'lint', render: cliRenderContext() });
    return { file: relPath, line: 1, rule: 'fix-not-writable', fixable: false, code: 'fix_not_writable', reason: errno.toLowerCase(),
      message: envelope.message, ...(envelope.fix ? { fix: envelope.fix } : {}), docs: docsUrl(ERROR_CATALOGUE.fix_not_writable.docs) };
  }
}

function refusalReason(error: OperationError): string {
  if (error.code === 'revision_conflict') return 'revision_changed';
  if (heldFileDiagnostic(error.message)) return 'held_file';
  return error.detail ?? writeFailureDiagnostic(error.code, error.message).reason;
}

function changedWhileLinting(sourceId: string, slug: string): OperationError {
  return opError('revision_conflict', 'The page changed while lint ran.',
    `Page ${slug} in '${sourceId}' changed after lint read it, so its repair was not submitted; the next cycle lints the current revision.`,
    { fix: readFix(`Shows page ${slug} in source ${sourceId} as it is now, with its revision.`, { argv: ['gbrain', 'get', '--source', sourceId, '--', slug] }) });
}

/**
 * #5180: publish one lint repair through the persistence coordinator, the
 * only writer a managed brain's worktree accepts. Resolves the page's slug
 * the way sync does and reads its revision BEFORE the bytes the repair is
 * built from, so the repair only ever replaces what it read: the file must
 * hold that revision (the coordinator's own canonical comparison) and the
 * repair is admitted against it. Returns 'published' once the coordinator
 * committed it (DB row and file in one guarded write), null when the file
 * no longer needs a repair, or a non-fixable issue when the repair must
 * wait: the file is not an indexed page of the source, the file diverged
 * from the database (the coordinator's refusal travels unchanged: its reason
 * and its fix), or the page changed between scan and admission. Any other
 * failure propagates and fails the run.
 */
async function publishLintFix(engine: BrainEngine, authority: MaintenanceAuthority, root: string, page: string,
  activePack: ParseOpts['activePack']): Promise<LintIssue | 'published' | null> {
  const relPath = relative(root, page);
  const slug = pathToSlug(relPath);
  const sourceId = authority.writer.sourceId;
  const snapshot = await engine.readPageSnapshot(slug, { sourceId, includeDeleted: true });
  if (!snapshot || snapshot.page.deleted_at) {
    return pendingIssue(relPath, slug, 'not_indexed', null,
      `fix not applied: ${slug} is not an indexed page of source ${sourceId} (a managed brain only rewrites indexed pages)`);
  }
  const scanned = existsSync(page) ? readSourceFileSync(page, 'utf-8') : null;
  const fixed = scanned === null ? null : fixContent(scanned);
  if (scanned === null ? !authority.binding : fixed === scanned) return null;
  const importPath = async () => scannerSourcePath(root, page, await scannerSlugRootMode(engine, sourceId, root));
  if (authority.binding && (scanned === null || !await fileMatchesSnapshot(engine, slug, scanned, snapshot, importPath, activePack))) {
    const current = await engine.readPageSnapshot(slug, { sourceId, includeDeleted: true });
    if (current?.revision !== snapshot.revision) return pendingIssue(relPath, slug, 'revision_changed', changedWhileLinting(sourceId, slug));
    try {
      await prepareFileTarget(engine, { source_id: sourceId, worktree_id: authority.binding.worktree_id, slug }, snapshot, fixed, undefined, { activePack });
    } catch (e) {
      if (e instanceof OperationError) return pendingIssue(relPath, slug, refusalReason(e), e);
      throw e;
    }
    // No refusal: the file now holds the scanned revision again (or the page has no file to repair).
    return scanned === null ? null : pendingIssue(relPath, slug, 'revision_changed', changedWhileLinting(sourceId, slug));
  }
  try {
    await publishMaintenancePage(engine, authority, slug, fixed!, { expectedRevision: snapshot.revision });
    return 'published';
  } catch (e) {
    // An admitted request the coordinator refused at its own file or revision check; a source-level
    // refusal before admission (no receipt) still fails the run.
    if (e instanceof OperationError && (e.code === 'revision_conflict' || e.code === 'source_changed' && e.writeRequest)) {
      return pendingIssue(relPath, slug, refusalReason(e), e);
    }
    throw e;
  }
}

/**
 * Library-level lint. Throws on validation errors (missing target, target
 * not found); lints otherwise. Does NOT print human-readable details (the
 * CLI wrapper handles that) — returns counts so Minions handlers can
 * report structured results. Safe from the worker — no process.exit.
 */
export async function runLintCore(opts: LintOpts): Promise<LintResult> {
  if (!opts.target) {
    throw new Error('lint: target (dir|file.md) required');
  }
  if (!existsSync(opts.target)) {
    throw new Error(`Not found: ${opts.target}`);
  }

  // #5180: a managed brain's worktree belongs to the persistence coordinator —
  // the legacy filesystem lock (and its writer guard) would refuse the whole
  // run, so lint reads the tree lock-free there and publishes fixes through
  // the coordinator below. Unmanaged brains keep the lock + direct writes.
  const managed = !!opts.engine && await isManagedBrain(opts.engine);
  if (!managed && opts.engine && opts.fix && !opts.dryRun && !hasSourceFilesystemLock(opts.target)) {
    return withSourceFilesystemLock(opts.engine, opts.target, () => runLintCore(opts), { signal: opts.signal });
  }

  const isSingleFile = statSync(opts.target).isFile();
  const pages = isSingleFile ? [opts.target] : collectPages(opts.target, opts.exclude ?? []);
  opts.onPagesCollected?.(pages.length);

  // Resolve content-sanity config and the active pack once for this lint
  // run (D1: lift DB config when reachable). Callers can pre-pass both
  // (opts.contentSanity + opts.typePack) to bypass the engine probe.
  const runtime = opts.contentSanity !== undefined && opts.typePack !== undefined
    ? null
    : await resolveLintContentSanity(opts.engine);
  const lintOpts: LintContentOpts = {
    contentSanity: opts.contentSanity ?? runtime?.contentSanity,
    typePack: opts.typePack !== undefined ? opts.typePack : runtime?.typePack ?? null,
  };

  // Durability-hardened brains (`gbrain sources harden`) promise every write
  // is committed AND pushed. An uncommitted lint repair would otherwise sit
  // as drift the cycle's sync phase flags and the post-commit hook never
  // pushes. Same gate + path-limited helper as put_page write-through
  // (#2426); covers the cycle, the lint/lint-fix minion handlers and the CLI
  // alike. Known ceiling: one commit + one backgrounded hook spawn per
  // repaired page on a large first run (the hook's flock coalesces pushes on
  // Linux; on macOS a losing concurrent push can log a spurious LOCAL-ONLY
  // line even though the later push landed) — a single end-of-run
  // multi-path commit would need a new helper.
  const repoProbe = isSingleFile ? dirname(opts.target) : opts.target;
  const applyFixes = !!opts.fix && !opts.dryRun;
  // #5180: on a managed brain the coordinator commits what it publishes, so
  // the durability commit stays on the legacy path only.
  const commitFixes = applyFixes && !managed && isDurabilityHardened(repoProbe);
  // #5180: the maintenance authority for coordinator writes, resolved once per
  // run like the synthesize/patterns phases. Throws `owner_unavailable` when
  // the managed source has no active canonical owner, which the cycle reports
  // as the phase's failure instead of a per-page filesystem refusal.
  const maintenance: MaintenanceAuthority | null = applyFixes && managed
    ? await maintenancePreflight(opts.engine!, opts.sourceId ?? 'default', isSingleFile ? undefined : opts.target)
    : null;
  // #5180: a single-file target on a managed brain derives its slug from the owned worktree root.
  // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- local_path/relative_path are the operator's own worktree binding rows (persistence_host_bindings / persistence_source_bindings) that maintenancePreflight just validated with assertPhysicalRoot; the join reproduces the binding root the coordinator itself uses (prepared-maintenance.ts) and only feeds relative() for slug derivation — no filesystem operation
  const managedRoot = !isSingleFile ? opts.target : maintenance?.binding?.local_path ? join(maintenance.binding.local_path, maintenance.binding.relative_path) : dirname(opts.target);

  let totalIssues = 0;
  let totalFixable = 0;
  let totalFixed = 0;
  let pagesWithIssues = 0;
  let fixPending = 0;
  const pendingIssues: LintIssue[] = [];
  // The coordinator parses files with the source's active pack; the divergence check must too.
  const activePack = maintenance?.binding
    ? (await loadActivePackForEngine(opts.engine!, { remote: false, sourceId: maintenance.writer.sourceId }).catch(() => null))?.manifest
    : undefined;

  for (let idx = 0; idx < pages.length; idx++) {
    assertSourceFilesystemActive();
    const page = pages[idx];
    // #1972: every 200 pages, yield to the event loop and honor abort. The
    // yield is what lets the abort signal actually fire (the rest of the loop
    // is synchronous); the break returns a valid partial LintResult since each
    // page is independently read + written.
    if (idx > 0 && idx % 200 === 0) {
      if (isAborted(opts.signal)) break;
      await new Promise<void>((resolve) => setImmediate(resolve));
    }
    const relPath = isSingleFile ? page : relative(opts.target, page);
    let content: string;
    try {
      content = readSourceFileSync(page, 'utf-8');
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
      // #5180: a managed --fix run asks the coordinator, which reports the page's missing canonical file as pending.
      const pending = maintenance ? await publishLintFix(opts.engine!, maintenance, managedRoot, page, activePack) : null;
      const issue = pending && pending !== 'published' ? pending : removedDuringScanIssue(relPath, opts.target);
      opts.onPageScanned?.();
      pagesWithIssues++;
      totalIssues++;
      if (issue.code === 'managed_write_pending') {
        fixPending++;
        if (pendingIssues.length < MAX_PENDING_ISSUES) pendingIssues.push(issue);
      }
      opts.onPageIssues?.(relPath, [issue], 0);
      continue;
    }
    const issues = lintContent(content, relPath, lintOpts);
    opts.onPageScanned?.();
    if (issues.length === 0) continue;
    pagesWithIssues++;
    totalIssues += issues.length;
    totalFixable += issues.filter(i => i.fixable).length;

    let fixCount = 0;
    if (opts.fix && issues.some(i => i.fixable)) {
      const fixed = fixContent(content);
      if (fixed !== content) {
        const fixable = issues.filter(i => i.fixable).length;
        if (!applyFixes) {
          fixCount = fixable;
        } else if (maintenance) {
          // #5180: managed brain — the coordinator rewrites the page (DB row
          // and worktree file together) or says why the repair must wait.
          const outcome = await publishLintFix(opts.engine!, maintenance, managedRoot, page, activePack);
          if (outcome === 'published') {
            fixCount = fixable;
          } else if (outcome) {
            issues.push(outcome);
            totalIssues++;
            fixPending++;
            if (pendingIssues.length < MAX_PENDING_ISSUES) pendingIssues.push(outcome);
          }
        } else {
          assertSourceFilesystemActive();
          const unwritable = writeLintFixOrRefusal(page, relPath, fixed);
          if (unwritable) {
            issues.push(unwritable);
            totalIssues++;
            fixPending++;
            if (pendingIssues.length < MAX_PENDING_ISSUES) pendingIssues.push(unwritable);
          } else {
            if (commitFixes) commitWriteThroughFile(repoProbe, page, relative(repoProbe, page).replace(/\.md$/u, ''));
            fixCount = fixable;
          }
        }
        totalFixed += fixCount;
      }
    }
    opts.onPageIssues?.(relPath, issues, fixCount);
  }

  return {
    pages_scanned: pages.length,
    pages_with_issues: pagesWithIssues,
    total_issues: totalIssues,
    total_fixable: totalFixable,
    total_fixed: totalFixed,
    dryRun: !!opts.dryRun,
    applied_fix: !!opts.fix,
    write_path: !applyFixes ? 'none' : maintenance ? 'coordinator' : 'filesystem',
    fix_pending: fixPending,
    pending_issues: pendingIssues,
  };
}

export async function runLint(args: string[]) {
  // #2649: --exclude=a,b or --exclude a,b — extra basenames to skip.
  const extraExcludes: string[] = [];
  const skipIdx = new Set<number>();
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a.startsWith('--exclude=')) {
      extraExcludes.push(...a.slice('--exclude='.length).split(',').map(s => s.trim()).filter(Boolean));
    } else if (a === '--exclude' && i + 1 < args.length) {
      extraExcludes.push(...args[i + 1].split(',').map(s => s.trim()).filter(Boolean));
      skipIdx.add(i + 1);
    }
  }
  const target = args.find((a, i) => !a.startsWith('--') && !skipIdx.has(i));
  const doFix = args.includes('--fix');
  const dryRun = args.includes('--dry-run');

  if (!target) {
    console.error('Usage: gbrain lint <dir|file.md> [--fix] [--dry-run] [--exclude a,b]');
    console.error('  --fix      Auto-fix fixable issues (LLM preambles, code fences)');
    console.error('  --dry-run  Preview fixes without writing');
    console.error('  --exclude  Comma-separated dir/file basenames to skip (in addition to node_modules)');
    process.exit(1);
  }

  if (!existsSync(target)) {
    console.error(`Not found: ${target}`);
    process.exit(1);
  }

  // W0 fix-wave (Tier-1 #14): ONE scan. Pre-fix this function ran its own
  // full read+lint+fix loop for human output and THEN called runLintCore for
  // the summary — every page linted twice, and with --fix the second pass
  // saw already-fixed files so the summary reported "0 auto-fixed" after
  // fixing N. Human detail now streams from runLintCore's per-page hooks
  // and the counts come from the same single pass.
  // Progress on stderr. Stdout keeps the per-issue human output it always had.
  // Ship-review perf catch: the tree is walked ONCE — runLintCore reports the
  // collected count via onPagesCollected (pre-fix the CLI ran its own
  // collectPages just to size the progress bar, a second full readdir/stat
  // walk on every directory lint).
  const { createProgress } = await import('../core/progress.ts');
  const { getCliOptions, cliOptsToProgressOptions } = await import('../core/cli-options.ts');
  const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));

  const result = await runLintCore({
    target,
    fix: doFix,
    dryRun,
    exclude: extraExcludes,
    onPagesCollected: (count) => progress.start('lint.pages', count),
    onPageScanned: () => progress.tick(1),
    onPageIssues: (relPath, issues, fixedCount) => {
      console.log(`\n${relPath}:`);
      for (const issue of issues) {
        const fixLabel = issue.fixable ? ' [fixable]' : '';
        console.log(`  L${issue.line} ${issue.rule}: ${issue.message}${fixLabel}`);
        if (issue.fix?.command) console.log(`    Fix: ${issue.fix.command}`);
        if (issue.docs) console.log(`    Docs: ${issue.docs}`);
      }
      if (fixedCount > 0) {
        console.log(`  ${dryRun ? '(dry run) ' : ''}Fixed ${fixedCount} issue(s)`);
      }
    },
  });

  progress.finish();
  console.log(`\n${result.pages_scanned} pages scanned. ${result.total_issues} issue(s) in ${result.pages_with_issues} page(s).`);
  if (doFix) {
    console.log(`${dryRun ? '(dry run) ' : ''}${result.total_fixed} auto-fixed.`);
  } else if (result.total_fixable > 0) {
    // #3958: only advertise --fix when at least one finding is actually
    // fixable — an all-unfixable report used to send operators on a no-op
    // `--fix` run that changed nothing.
    console.log(`Run with --fix to auto-fix ${result.total_fixable} fixable issue(s).`);
  }
}