Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.9 KB
Raw Blame
// src/core/extract-ner.ts
// v0.41.18.0 (A10, T7). NER link extraction reuses the by-mention gazetteer
// and applies schema-pack `link_types[].inference.regex` patterns to assign
// a typed link verb ("CEO of Acme" → 'works_at' linking the page to Acme).
//
// Codex finding #12 design (locked): do NOT split link_source='ner' as a
// new provenance — that would break every existing link_source='mentions'
// query (backlink-count filter, orphan-ratio, doctor checks). Instead:
// keep link_source='mentions' AND set link_kind='typed_ner' on the new row
// (v98 added the column). Legacy plain mentions stay link_kind=NULL
// (semantically 'plain').
//
// The links UNIQUE constraint excludes link_kind, so an existing plain
// mention row + a typed_ner row for the same (from, to, type, source, origin)
// collide — DO NOTHING. NER does NOT overwrite plain mentions; the verb
// link goes in as a different row with a different link_type.

import type { BrainEngine } from './engine.ts';
import type { LinkBatchInput } from './engine.ts';
import { buildGazetteer, findMentionedEntities, type Gazetteer } from './by-mention.ts';
import { isCrossSourceLinksEnabled } from './link-extraction.ts';
import { inferLinkTypeFromPack } from './schema-pack/link-inference.ts';
import { loadActivePackForLocalEngine, packSupportsNerInference } from './schema-pack/best-effort.ts';
import { isQuarantined } from './quarantine.ts';
import { renderFragment, sqlFragment } from './engine-sql/fragment.ts';
import { entityTypePredicateSql, resolveEntityTypes } from './schema-pack/entity-types.ts';

export interface ExtractNerOpts {
  /** When true: enumerate but don't write. */
  dryRun?: boolean;
  /** Optional source-id filter on the WALK (gazetteer stays brain-wide). */
  sourceIdFilter?: string;
  /** Optional page-type filter on the WALK. */
  typeFilter?: string;
  /** Only scan pages with updated_at after this ISO date. */
  since?: string;
  /**
   * Pre-built gazetteer (T7+: combined `--by-mention --ner` walk shares
   * one gazetteer across both passes). When omitted, this fn builds its own.
   */
  gazetteer?: Gazetteer;
  /** Optional progress hook called per processed page. */
  onProgress?: (done: number, total: number, created: number) => void;
}

export interface ExtractNerResult {
  /** Pages scanned. */
  pages: number;
  /** Typed-NER links created (or would-have-created in dry-run). */
  created: number;
  /** The active schema pack had no NER inference rules, or a pack in scope did not load (entity types unknown). */
  pack_unavailable: boolean;
}

/** Context window scanned around each mention for verb-pattern matching. */
const CONTEXT_WINDOW_CHARS = 80;

/**
 * Pure helper: get the context window around a mention's character offset.
 * Returns the substring [offset - W, offset + name.length + W] of the body.
 * Caller passes (body, offset, name.length).
 */
export function getContextWindow(
  body: string,
  offset: number,
  nameLen: number,
  window: number = CONTEXT_WINDOW_CHARS,
): string {
  const start = Math.max(0, offset - window);
  const end = Math.min(body.length, offset + nameLen + window);
  return body.slice(start, end);
}

/**
 * Pure helper: derive the entity-type→link-verb pair from a single mention.
 * Returns null when (a) target type unknown, (b) pack has no inference for
 * that type, (c) no verb pattern matches the surrounding context.
 *
 * Exported for unit tests; the orchestrator below uses it directly.
 */
export function inferNerLinkType(
  pack: Parameters<typeof inferLinkTypeFromPack>[0],
  targetType: string | undefined,
  context: string,
): string | null {
  if (!targetType) return null;
  try {
    return inferLinkTypeFromPack(pack, targetType, context, undefined, undefined, { ner: true });
  } catch {
    return null;
  }
}

/**
 * extractNerLinks: walk pages, find body mentions, apply schema-pack
 * inference regex per (target_type, surrounding context) to assign a typed
 * link verb. Returns count of created links.
 *
 * Best-effort wrt the schema pack: if no active pack OR no link_types
 * declared OR no inference.regex on any link_type, the function returns
 * pack_unavailable=true and 0 created. Caller (CLI / handler) surfaces a
 * one-line hint instead of an error.
 */
export async function extractNerLinks(
  engine: BrainEngine,
  opts: ExtractNerOpts = {},
): Promise<ExtractNerResult> {
  const dryRun = opts.dryRun ?? false;

  // Pack best-effort: no pack, or no link_type with an inference.regex → NER
  // has no patterns to match and we'd waste a full walk. `!pack` first so TS
  // narrows pack to non-null for the rest of the function; packSupportsNerInference
  // (shared with the onboard recommender) covers the no-link-types / no-regex cases.
  //
  // Resolves via loadActivePackForLocalEngine, the SAME path the onboard
  // recommender uses, so the two cannot resolve different packs and then
  // disagree about whether this job can help. The previous
  // `loadActivePackBestEffort({ engine } as never)` shape ran this local job
  // under REMOTE trust gating (that helper defaults `remote: ctx.remote ?? true`)
  // and could not see a DB-side pack flip.
  const pack = await loadActivePackForLocalEngine(engine);
  if (!pack || !packSupportsNerInference(pack)) {
    return { pages: 0, created: 0, pack_unavailable: true };
  }

  const gazetteer = opts.gazetteer ?? await buildGazetteer(engine);
  if (gazetteer.size === 0) {
    return { pages: 0, created: 0, pack_unavailable: false };
  }
  const allowCrossSource = await isCrossSourceLinksEnabled(engine);

  // Pre-fetch target entity types so inferLinkType has the type signal
  // without an N+1 getPage round-trip. Pulls the slug→type map from
  // listAllPageRefs + a single listPages projection.
  const targetTypeMap = await buildTargetTypeMap(engine);
  if (!targetTypeMap) return { pages: 0, created: 0, pack_unavailable: true };

  const allRefs = opts.sourceIdFilter
    ? (await engine.listAllPageRefs()).filter((r) => r.source_id === opts.sourceIdFilter)
    : await engine.listAllPageRefs();

  let processed = 0;
  let created = 0;
  const batch: LinkBatchInput[] = [];
  const BATCH_SIZE = 500;
  const sinceMs = opts.since ? new Date(opts.since).getTime() : null;

  async function flush() {
    if (batch.length === 0) return;
    if (!dryRun) {
      try {
        created += await engine.addLinksBatch(batch); // gbrain-allow-direct-insert: extract-ner — typed NER link write
      } catch {
        // batch error: drop; the per-page progress continues
      }
    } else {
      created += batch.length;
    }
    batch.length = 0;
  }

  for (const { slug, source_id } of allRefs) {
    const page = await engine.getPage(slug, { sourceId: source_id });
    if (!page || isQuarantined(page.frontmatter)) continue;
    if (opts.typeFilter && page.type !== opts.typeFilter) continue;
    if (sinceMs !== null) {
      const updatedMs = new Date(page.updated_at).getTime();
      if (Number.isFinite(updatedMs) && updatedMs <= sinceMs) continue;
    }
    processed++;
    opts.onProgress?.(processed, allRefs.length, created);

    const body = page.compiled_truth + '\n\n' + (page.timeline ?? '');
    if (!body.trim()) continue;

    const mentions = findMentionedEntities(body, gazetteer, {
      fromSlug: slug,
      fromSourceId: source_id,
      allowCrossSource,
    });
    if (mentions.length === 0) continue;

    for (const m of mentions) {
      const targetType = targetTypeMap.get(`${m.source_id}::${m.slug}`);
      const context = getContextWindow(body, m.offset, m.name.length);
      const verb = inferNerLinkType(pack.manifest, targetType, context);
      if (!verb) continue;

      batch.push({
        from_slug: slug,
        to_slug: m.slug,
        link_type: verb,
        link_source: 'mentions',
        link_kind: 'typed_ner',
        context: m.name,
        from_source_id: source_id,
        to_source_id: m.source_id,
      });
      if (batch.length >= BATCH_SIZE) await flush();
    }
  }

  await flush();
  return { pages: processed, created, pack_unavailable: false };
}

/**
 * Helper: build a Map<sourceId::slug → type> for the entity-typed pages, by
 * the active pack(s) per source (#4772, the get_health predicate bound in
 * SQL). One round-trip; targets cached at extraction-start so
 * inferNerLinkType doesn't pay an N+1 cost per mention. Null when a pack in
 * scope did not load (entity types unknown); an engine error throws.
 */
async function buildTargetTypeMap(engine: BrainEngine): Promise<Map<string, string> | null> {
  const entityTypes = await resolveEntityTypes(engine, null);
  if (entityTypes.status === 'pack_unavailable') return null;
  const sql = renderFragment(sqlFragment`SELECT p.slug, p.source_id, p.type FROM pages p
    WHERE ${entityTypePredicateSql('p', entityTypes.filter)} AND p.deleted_at IS NULL`);
  const rows = await engine.executeRaw<{ slug: string; source_id: string; type: string }>(sql.text, sql.params);
  return new Map(rows.map((row) => [`${row.source_id}::${row.slug}`, row.type]));
}