Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
34.8 KB
Raw Blame
/**
 * Enrichment as a global service.
 *
 * Shared library callable from any ingest pathway. Handles the brain CRUD
 * for entity enrichment: check brain, create/update page, backlink, timeline.
 *
 * External API enrichment (people data APIs, professional networks) remains
 * agent-orchestrated per the enrich skill file. This library handles the
 * brain-side operations.
 *
 * Entity mention counts are derived from engine.searchKeyword() on the
 * existing data (clamped to 100 results). Source tracking derives from
 * page type/slug prefix since SearchResult has no metadata.skill field.
 */

import type { BrainEngine } from './engine.ts';
import type { OperationContext } from './ops/contract.ts';
import { OperationError } from './ops/contract.ts';
import { waitForCapacity } from './backoff.ts';
import { quarantineMarkers } from './extraction-review.ts';
import { SLUG_WORD_CHARS, SLUG_VARIATION_SELECTORS_RE } from './cjk.ts';
import { createHash } from 'node:crypto';
import { portableSegment, RESERVED_DEVICE_NAME, segmentByteBudget, truncateUtf8 } from './portable-slug.ts';
// #3994: created stubs route through serializeMarkdown + importFromContent
// (the same parse→chunk→embed pipeline put_page uses) instead of a bare
// engine.putPage, so fresh entity pages land in the retrieval surface
// (content_chunks + embeddings) — the #2163 concept-page precedent.
import { importFromContent } from './import-file.ts';
import { serializeMarkdown } from './markdown.ts';
import { isAvailable } from './ai/gateway.ts';
// #4222: shared generic-token reject list — same list gates the by-mention
// gazetteer and drives the junk_entity_hubs doctor check.
import { isJunkEntityName } from './entity-name-quality.ts';
import { followRetirement } from './entities/resolve.ts';
import { maintenanceTransaction } from './persistence/attribution.ts';
import { managedPersistenceEnabled } from './persistence/ownership.ts';
import { remoteReadPolicySql, remoteTargetHidden } from './persistence/page-visibility.ts';
import { resolveExcludePrivatePages } from './search/private-visibility.ts';
import { assertUnmanagedCanonicalWriter } from './persistence/maintenance.ts';
import { deriveTrust, derivedMaintenanceTransaction, lowerDerivedPage, lowerDerivedPageIn, lowerToDerivedTier } from './trust/taint.ts';
import { derivedGateConfig, derivedGateInput, recordTimelineFlag, timelineRowAllowed } from './trust/derived-gate.ts';
import { assessTimelineForGate } from './write-gate.ts';
import { renderTimelineEntry, spliceTimelineBlock } from './timeline-write-through.ts';

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

export interface EnrichmentRequest {
  entityName: string;
  entityType: 'person' | 'company';
  context: string;
  sourceSlug: string;
  tier?: 1 | 2 | 3;
}

/**
 * Trust options for the enrichment write path (issue #160).
 *
 * `trusted: true` — the input text comes from the machine owner via the
 * trusted local CLI (ctx.remote === false) AND the caller passed an explicit
 * opt-in flag. Stubs write direct as authoritative entity pages.
 *
 * Anything else (undefined, false, absent) is UNTRUSTED — fail-closed,
 * mirroring the OperationContext.remote invariant ("anything not strictly
 * false is remote"). Created stubs land in the quarantine lane: frontmatter
 * `provenance: 'auto-extracted'` + `status: 'unverified'`. They are excluded
 * from authoritative retrieval boosts and wait in the review queue
 * (`extraction_pending` / `extraction_review` ops) until the owner promotes
 * or rejects them.
 */
export interface EnrichmentTrustOptions {
  trusted?: boolean;
  /** Source to read/write in (multi-source brains). Omitted → engine default. */
  sourceId?: string;
  /** #6262: the caller's context; on a managed brain every write goes through the coordinator under its authority. */
  ctx?: OperationContext;
  /** GBRA-431: the source page exists but the remote caller cannot read it, so it is treated exactly as a missing page (no taint read from it, no backlink to it). */
  sourceHidden?: boolean;
}

export interface EnrichmentResult {
  slug: string;
  action: 'created' | 'updated' | 'skipped';
  /** True when the created stub landed in the quarantine lane (issue #160). */
  quarantined?: boolean;
  /**
   * Why nothing was written: the page is retired and its superseded_by / merged_into chain has no live target (B-9),
   * or both this entity's slug and its hash-tagged slug hold pages that name another entity.
   */
  skipped_reason?: 'retired_page_unfollowable' | 'entity_slug_taken' | 'page_not_found';
  tier: 1 | 2 | 3;
  backlinkCreated: boolean;
  timelineAdded: boolean;
  mentionCount: number;
  mentionSources: string[];
  suggestedTier: 1 | 2 | 3;
  tierEscalated: boolean;
}

// ---------------------------------------------------------------------------
// Entity naming utilities
// ---------------------------------------------------------------------------

// Keep-set mirrors sync.ts:slugifySegment (single grammar, see cjk.ts
// docstring) so an entity minted here and a file synced under brain/people/
// never diverge on what counts as a slug character.
const SLUGIFY_ENTITY_KEEP_RE = new RegExp(`[^${SLUG_WORD_CHARS}]`, 'gu');

/**
 * Convert an entity name to a URL-safe slug.
 *
 * Was ASCII-only ([a-z0-9]), which silently dropped every non-Latin
 * character — Cyrillic/CJK/Arabic/etc. names slugified to '' or a bare
 * '-', producing empty or colliding people/companies slugs. Now mirrors
 * sync.ts:slugifySegment's Unicode-aware keep-set: strip Latin accents to
 * their base letter (fold, don't drop), keep every script's own letters
 * as-is (Cyrillic, CJK, Devanagari, …). A name that slugifies to a Windows
 * reserved device name gets the entity type as a suffix (`Con` →
 * `people/con-person`) and an over-long one is cut to the file-name budget
 * and tagged with a hash of its full slug (portable-slug.ts), so a stub is
 * always a page export and write-through can store and two long names sharing
 * a prefix keep separate pages; a page an older brain created as `people/con`
 * is still found through the legacy-slug lookup in enrichEntity.
 */
export function slugifyEntity(name: string, type: 'person' | 'company'): string {
  return `${type === 'person' ? 'people' : 'companies'}/${entitySegment(entitySlugBase(name), type, false)}`;
}

/** The full slug segment a name maps to before portability escaping: the identity two names share when it is equal. */
function entitySlugBase(name: string): string {
  return name
    .replace(/['‘’]/g, '')
    .normalize('NFD')
    .replace(/[̀-ͯ]/g, '')
    .normalize('NFC')
    .replace(SLUG_VARIATION_SELECTORS_RE, '')
    .toLowerCase()
    .replace(SLUGIFY_ENTITY_KEEP_RE, '-')
    .replace(/-+/g, '-')
    .replace(/^-+|-+$/g, '');
}

/**
 * The portable file name for a slug base: a reserved name gets the entity type
 * as a suffix, and a name over the budget (or any name when `tagged`) ends in
 * the first 8 hex digits of the base's SHA-256, cut to fit before the tag.
 */
function entitySegment(base: string, type: 'person' | 'company', tagged: boolean): string {
  const renamed = base.replace(RESERVED_DEVICE_NAME, name => `${name}-${type}`);
  const budget = segmentByteBudget(true);
  if (!tagged && Buffer.byteLength(renamed) <= budget) return renamed;
  const tag = createHash('sha256').update(base).digest('hex').slice(0, 8);
  return `${truncateUtf8(renamed, budget - tag.length - 1).replace(/[-._:]+$/, '')}-${tag}`;
}

/**
 * The slug the pre-Unicode slugifier produced for the same name (ASCII only:
 * every non-[a-z0-9] run became '-'). Kept so enrichment can find entity pages
 * that existing brains created before slugifyEntity learned other scripts,
 * instead of minting a second page for the same entity.
 */
export function legacyAsciiEntitySlug(name: string, type: 'person' | 'company'): string {
  const slug = name
    .toLowerCase()
    .replace(/['']/g, '')
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-+|-+$/g, '');
  const prefix = type === 'person' ? 'people' : 'companies';
  return `${prefix}/${slug}`;
}

/** Get the brain page path for an entity. */
export function entityPagePath(name: string, type: 'person' | 'company'): string {
  return slugifyEntity(name, type);
}

// ---------------------------------------------------------------------------
// Core enrichment
// ---------------------------------------------------------------------------

/**
 * Enrich a single entity: check brain, create/update, backlink, timeline.
 * Uses searchKeyword to count mentions and derive source skills.
 */
export async function enrichEntity(
  engine: BrainEngine,
  request: EnrichmentRequest,
  opts?: EnrichmentTrustOptions,
): Promise<EnrichmentResult> {
  const step = await planEntity(engine, request, opts);
  if (!('managed' in step)) return step;
  const [written] = await writeManagedEntities(engine, [step]);
  if ('error' in written!) throw written.error;
  return written!;
}

/**
 * G6: a managed entity's one page write, planned before it is submitted: a new stub (its first timeline bullet inside
 * its markdown when the gate allows the row outright) or the bullet for an existing page. A stub whose bullet the gate
 * flags takes the bullet as a second write (add_timeline_entry records the flag). `result` is the reply before the write.
 */
interface ManagedEntity {
  managed: true;
  result: EnrichmentResult;
  ctx: OperationContext;
  derivation: Awaited<ReturnType<typeof deriveTrust>>;
  sourceId: string;
  sourceSlug: string;
  stub: { frontmatter: Record<string, unknown>; content: string; type: 'person' | 'company'; title: string } | null;
  entry: { date: string; summary: string; source: string } | null;
  /** The stub's markdown carries the bullet (the page add_timeline_entry would have left after it). */
  inline: boolean;
}

/** Everything enrichEntity decides and, on an unmanaged brain, writes; a managed entity comes back as its planned write. */
async function planEntity(
  engine: BrainEngine,
  request: EnrichmentRequest,
  opts?: EnrichmentTrustOptions,
): Promise<EnrichmentResult | ManagedEntity> {
  const base = entitySlugBase(request.entityName);
  const prefix = request.entityType === 'person' ? 'people' : 'companies';
  const candidateSlug = `${prefix}/${entitySegment(base, request.entityType, false)}`;
  const sourceId = opts?.sourceId ?? 'default';
  let slug = await engine.resolveSlugWithAlias(candidateSlug, sourceId);
  // Fail-closed: only an explicit `trusted: true` writes authoritative pages.
  const trusted = opts?.trusted === true;
  const scope = opts?.sourceId ? { sourceId: opts.sourceId } : undefined;
  // #5575 I2: a stub and its timeline row restate the source page's mention, so they carry its taint (capped at agent_written).
  const derivation = await deriveTrust(engine, [{ table: 'pages', sourceId, slug: request.sourceSlug, hidden: opts?.sourceHidden === true }], { channel: 'derive:enrichment' });
  const ctx = opts?.ctx;
  // #6262: a managed brain refuses direct writes, so each one goes through the coordinator under the caller's authority.
  const managedBrain = await managedPersistenceEnabled(engine);
  const managed = ctx !== undefined && managedBrain;
  // G6: the direct branch below writes the database first (and falls back to a raw page write), so a git-canonical source refuses here.
  if (!managedBrain) await assertUnmanagedCanonicalWriter(engine, 'extract_entities', { sourceId });
  // 1. Count existing mentions for tier auto-escalation
  // GBRA-431: a remote caller's count covers only the pages it can read.
  const { mentionCount, mentionSources } = await countMentions(engine, request.entityName, opts?.sourceId,
    ctx && ctx.remote !== false && await resolveExcludePrivatePages(engine, ctx.remote));

  // 2. Determine tier (auto-escalate based on mentions)
  const suggestedTier = suggestTier(mentionCount, mentionSources, request.context);
  const tier = request.tier || suggestedTier;
  const tierEscalated = suggestedTier < (request.tier || 3); // lower tier number = higher importance

  // 3. Check if entity page exists
  // GBRA-431: a remote caller never writes onto, or learns anything from, an entity page it cannot read: the first
  // such page met while resolving the slug skips this entity as not found, under the slug its name maps to.
  const hidden = async (at: string, page: unknown) => page !== null && ctx !== undefined && await remoteTargetHidden(engine, ctx.remote, sourceId, at);
  const notFound = (): EnrichmentResult => ({ slug: candidateSlug, action: 'skipped', skipped_reason: 'page_not_found', tier, backlinkCreated: false,
    timelineAdded: false, mentionCount, mentionSources, suggestedTier, tierEscalated });
  let existingPage = await engine.getPage(slug, scope);
  if (await hidden(slug, existingPage)) return notFound();
  // F8: a page whose title maps to this slug base names this entity; any other page at the slug may name another one.
  const namesEntity = (title: string | undefined) => entitySlugBase(title ?? '') === base;
  if (!existingPage || !namesEntity(existingPage.title)) {
    // A page an older brain created under the ASCII-only slug is the same entity.
    const legacy = legacyAsciiEntitySlug(request.entityName, request.entityType);
    let legacyFound = false;
    if (legacy !== candidateSlug && !/^(people|companies)\/$/.test(legacy)) {
      const legacySlug = await engine.resolveSlugWithAlias(legacy, sourceId);
      const legacyPage = await engine.getPage(legacySlug, { sourceId });
      if (await hidden(legacySlug, legacyPage)) return notFound();
      // The legacy slug is lossy (José -> people/jos), so only reuse a page that names this entity.
      const same = (t: string) => t.normalize('NFC').trim().toLowerCase();
      if (legacyPage && same(legacyPage.title ?? '') === same(request.entityName)) { slug = legacySlug; existingPage = legacyPage; legacyFound = true; }
    }
    // An over-budget name an older gbrain minted at its cut slug with no hash tag; that slug is shared by every name with the same prefix.
    const untagged = `${prefix}/${portableSegment(base, true, request.entityType)}`;
    if (!legacyFound && !existingPage && untagged !== candidateSlug) {
      const untaggedSlug = await engine.resolveSlugWithAlias(untagged, sourceId);
      const untaggedPage = await engine.getPage(untaggedSlug, scope);
      if (await hidden(untaggedSlug, untaggedPage)) return notFound();
      if (untaggedPage && namesEntity(untaggedPage.title)) { slug = untaggedSlug; existingPage = untaggedPage; legacyFound = true; }
    }
    // An escaped slug (Aux -> companies/aux-company) can be another entity's natural slug, and a natural slug can be
    // the escaped slug another entity was minted at (Prn Company's companies/prn-company held by Prn): either way the
    // page names another entity, so this one takes its hash-tagged slug. A page a person titled differently at this
    // entity's natural slug stays this entity's page.
    const heldByOther = !legacyFound && existingPage !== null &&
      (candidateSlug !== `${prefix}/${base}` || slugifyEntity(existingPage.title ?? '', request.entityType) === candidateSlug);
    if (heldByOther) {
      const tagged = `${prefix}/${entitySegment(base, request.entityType, true)}`;
      const taggedSlug = tagged === candidateSlug ? null : await engine.resolveSlugWithAlias(tagged, sourceId);
      const taggedPage = taggedSlug === null ? null : await engine.getPage(taggedSlug, scope);
      if (taggedSlug !== null && await hidden(taggedSlug, taggedPage)) return notFound();
      if (taggedSlug === null || (taggedPage && !namesEntity(taggedPage.title))) {
        return { slug, action: 'skipped', skipped_reason: 'entity_slug_taken', tier, backlinkCreated: false, timelineAdded: false,
          mentionCount, mentionSources, suggestedTier, tierEscalated };
      }
      slug = taggedSlug;
      existingPage = taggedPage;
    }
  }
  // B-9: a retired page (canonical: false, superseded_by) answers through its canonical page, as fact resolution does (#6898).
  if (existingPage) {
    const live = await followRetirement(engine, sourceId, slug, ctx && ctx.remote !== false ? `AND ${await remoteReadPolicySql(engine, 'p')}` : '');
    if (!live) {
      return { slug, action: 'skipped', skipped_reason: 'retired_page_unfollowable', tier, backlinkCreated: false, timelineAdded: false,
        mentionCount, mentionSources, suggestedTier, tierEscalated };
    }
    if (live.slug !== slug) { slug = live.slug; existingPage = await engine.getPage(slug, { sourceId }); }
  }
  let action: 'created' | 'updated' | 'skipped';

  if (existingPage) {
    // UPDATE path — add timeline entry
    action = 'updated';
  } else {
    // #4222: refuse to MINT a page for a junk entity name — a single
    // generic token ("Will", "Info", "Chief", "Unknown") or a bare
    // @handle. These come from over-eager extractors and become
    // near-empty mega-hubs that accrete thousands of mention edges.
    // Existing pages are user-visible and stay trusted (CK12 precedent);
    // only creation is gated — no auto-delete, ever.
    if (isJunkEntityName(request.entityName)) {
      return {
        slug,
        action: 'skipped',
        tier,
        backlinkCreated: false,
        timelineAdded: false,
        mentionCount,
        mentionSources,
        suggestedTier,
        tierEscalated,
      };
    }
    // CREATE path — new entity page
    const title = request.entityName;
    const type = request.entityType;
    const content = generateStubContent(request.entityName, request.entityType, request.context);
    const frontmatter = {
      created: new Date().toISOString().split('T')[0],
      source: request.sourceSlug,
      tier,
      // issue #160 quarantine lane: stubs extracted from untrusted input
      // carry provenance + unverified markers until the owner reviews them.
      ...(trusted ? {} : quarantineMarkers()),
    };
    if (managed) {
      // No direct-write fallback: a refusal or a pending write reaches the caller as its envelope.
      return managedEntity(engine, { managed: true, ctx: ctx!, derivation, sourceId, sourceSlug: request.sourceSlug, stub: { frontmatter, content, type, title }, entry: null, inline: false,
        result: { slug, action: 'created', ...(!trusted ? { quarantined: true } : {}), tier, backlinkCreated: false, timelineAdded: false,
          mentionCount, mentionSources, suggestedTier, tierEscalated } }, request.context);
    } else try {
      // #3994: canonical import pipeline so the stub is chunked (+ embedded
      // when a provider is configured) and reachable by the recall arms.
      const md = serializeMarkdown(frontmatter, content, '', { type, title, tags: [] });
      await importFromContent(engine, slug, md, {
        noEmbed: !isAvailable('embedding'), writeGate: derivedGateInput(derivation.trust),
        ...(opts?.sourceId ? { sourceId: opts.sourceId } : {}),
      });
      await lowerDerivedPage(engine, derivation, sourceId, slug);
    } catch (e) {
      // Fail-open fallback: a pipeline error (parse edge case, size guard)
      // must never regress the batch — the pre-#3994 direct write still
      // produces a page (unchunked, but present + reviewable). Warn loudly:
      // silence here would hide that the stub is invisible to vector recall
      // until the next `gbrain embed --stale` / re-import sweep.
      process.stderr.write(
        `[enrich] import pipeline failed for stub ${slug} (${e instanceof Error ? e.message : String(e)}); ` +
        'falling back to a direct unchunked write — the page exists but is not chunked/embedded until re-imported.\n',
      );
      await derivedMaintenanceTransaction(engine, derivation, async tx => {
        const page = await tx.putPage(slug, { title, type, compiled_truth: content, timeline: '', frontmatter }, scope);
        return { result: page, rows: [{ table: 'pages', id: page.id, sourceId }] };
      });
    }
    action = 'created';
  }
  if (managed) return managedEntity(engine, { managed: true, ctx: ctx!, derivation, sourceId, sourceSlug: request.sourceSlug, stub: null, entry: null, inline: false,
    result: { slug, action, tier, backlinkCreated: false, timelineAdded: false, mentionCount, mentionSources, suggestedTier, tierEscalated } }, request.context);

  // 4. Add timeline entry
  let timelineAdded = false;
  try {
    const entry = timelineEntryFor(request.sourceSlug, request.context);
    // #5575 B3: the caller's context lands on the entity page, so it passes the write gate at the derived tier.
    const assessment = assessTimelineForGate(entry, derivedGateInput(derivation.trust), await derivedGateConfig(engine));
    const write = async (tx: BrainEngine) => {
      const added = await tx.addTimelineEntry(slug, entry, scope); // gbrain-allow-direct-insert: auto-timeline reconciliation triggered by entity reference in source markdown
      if (added) await recordTimelineFlag(tx, assessment, { slug, source_id: sourceId, ...entry });
      return true;
    };
    timelineAdded = timelineRowAllowed(assessment) && await maintenanceTransaction(engine, write, derivation.trust);
  } catch {
    // Timeline add failed (page might not support it)
  }

  // 5. Add backlink from entity to source
  let backlinkCreated = false;
  if (!opts?.sourceHidden) try {
    await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`, undefined, undefined, undefined, undefined, opts?.sourceId ? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId } : undefined); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdown
    backlinkCreated = true;
  } catch {
    // Link might already exist
  }

  return {
    slug,
    action,
    ...(action === 'created' && !trusted ? { quarantined: true } : {}),
    tier,
    backlinkCreated,
    timelineAdded,
    mentionCount,
    mentionSources,
    suggestedTier,
    tierEscalated,
  };
}

/**
 * Enrich multiple entities with throttling between each.
 * config.onProgress is called after each entity so callers can stream
 * progress to a reporter (CLI) or job.updateProgress (Minion).
 */
export async function enrichEntities(
  engine: BrainEngine,
  requests: EnrichmentRequest[],
  config?: { throttle?: boolean; onProgress?: (done: number, total: number, name: string) => void } & EnrichmentTrustOptions,
): Promise<EnrichmentResult[]> {
  const results: EnrichmentResult[] = [];
  // G6: a managed brain's entity writes are planned first and submitted together (writeManagedEntities), so Postgres
  // publishes them in groups. A page planned twice is written by the first plan, then planned again against its result.
  let pending: Array<{ step: ManagedEntity; name: string }> = [];
  const refused = (e: unknown, name: string) => {
    // #6262: a managed refusal stops the batch and says which entities were already written.
    if (e instanceof OperationError && results.length) {
      e.why = `${e.why ?? ''} ${results.length} of ${requests.length} entities were written before ${name}: ${results.map(r => r.slug).join(', ')}. Run extract_entities again on the same text once the write can commit; written entities are updated, not duplicated.`.trim();
    }
    return e;
  };
  const settle = (result: EnrichmentResult, name: string) => {
    results.push(result);
    config?.onProgress?.(results.length, requests.length, name);
  };
  const flush = async () => {
    const batch = pending;
    pending = [];
    const written = await writeManagedEntities(engine, batch.map(entry => entry.step));
    const failed = written.findIndex(entry => 'error' in entry);
    written.forEach((entry, i) => { if (!('error' in entry)) settle(entry, batch[i]!.name); });
    if (failed >= 0) throw refused((written[failed] as { error: unknown }).error, batch[failed]!.name);
  };
  for (const req of requests) {
    if (config?.throttle !== false) {
      await waitForCapacity({ maxAttempts: 5 }); // shorter timeout for batch items
    }
    const plan = () => planEntity(engine, req, { trusted: config?.trusted, sourceId: config?.sourceId, ctx: config?.ctx, sourceHidden: config?.sourceHidden });
    let step: EnrichmentResult | ManagedEntity;
    try {
      step = await plan();
      if ('managed' in step && pending.some(entry => entry.step.result.slug === (step as ManagedEntity).result.slug)) { await flush(); step = await plan(); }
    } catch (e) { throw refused(e, req.entityName); }
    if ('managed' in step) pending.push({ step, name: req.entityName });
    else settle(step, req.entityName);
    if (pending.length >= MANAGED_ENTITY_BATCH) await flush();
  }
  await flush();
  return results;
}

/** Entities a managed batch submits at once: put_pages' page cap (page-batch.ts PAGE_BATCH_MAX_PAGES). */
const MANAGED_ENTITY_BATCH = 50;

const timelineEntryFor = (sourceSlug: string, context: string) =>
  ({ date: new Date().toISOString().split('T')[0] ?? '', summary: `Referenced in [${sourceSlug}](${sourceSlug}) — ${context}`, source: sourceSlug });

/** Completes a managed entity's plan with its timeline bullet: the gate decides whether it is written and whether a stub carries it inline. */
async function managedEntity(engine: BrainEngine, step: ManagedEntity, context: string): Promise<ManagedEntity> {
  const entry = timelineEntryFor(step.sourceSlug, context);
  // #5575 B3: the caller's context lands on the entity page, so it passes the write gate at the derived tier.
  const assessment = assessTimelineForGate(entry, derivedGateInput(step.derivation.trust), await derivedGateConfig(engine));
  if (!timelineRowAllowed(assessment)) return step;
  return { ...step, entry, inline: step.stub !== null && assessment.verdict === 'allow' && renderTimelineEntry(entry, step.result.slug) !== null };
}

/** A stub's markdown; an inline bullet is spliced into its timeline, as add_timeline_entry would after it. */
function stubMarkdown(step: ManagedEntity): string {
  const { frontmatter, content, type, title } = step.stub!;
  const bullet = step.inline ? renderTimelineEntry(step.entry!, step.result.slug)!.block : null;
  return bullet ? serializeMarkdown(frontmatter, content.trimEnd(), spliceTimelineBlock('', step.entry!.date, bullet), { type, title, tags: [] })
    : serializeMarkdown(frontmatter, content, '', { type, title, tags: [] });
}

/**
 * G6: writes planned managed entities as page-mutation batches: every entity's first write (its stub, or its bullet),
 * then the bullets of stubs whose bullet the gate flagged. A stub's page is lowered to the derived tier after it lands;
 * an unknown-tier derivation lowers its new timeline row as on an unmanaged brain. G5: the bullet's [source](source) link
 * is the backlink, derived by the publication from the page's markdown. Each entity settles on its own.
 */
async function writeManagedEntities(engine: BrainEngine, steps: ManagedEntity[]): Promise<Array<EnrichmentResult | { error: unknown }>> {
  if (!steps.length) return [];
  const { submitPageMutations } = await import('./persistence/page-batch.ts');
  const { coordinatedDatabaseWrite } = await import('./persistence/database-write.ts');
  const ctx = steps[0]!.ctx;
  type PageWrite = { step: ManagedEntity; input: { operation: 'put_page' | 'add_timeline_entry'; params: Record<string, unknown> } };
  const origin = (step: ManagedEntity) => step.derivation.trust.tier === 'external_untrusted' ? { content_origin: 'tool_output' } : {};
  const bullet = (step: ManagedEntity): PageWrite => ({ step, input: { operation: 'add_timeline_entry', params: { slug: step.result.slug, ...step.entry!, ...origin(step) } } });
  const submit = async (writes: PageWrite[]) => {
    const replies = await submitPageMutations(ctx, writes.map(write => write.input));
    return new Map(writes.map((write, i) => [write.step, replies[i]!]));
  };
  const first = await submit(steps.flatMap((step): PageWrite[] => step.stub !== null
    ? [{ step, input: { operation: 'put_page', params: { slug: step.result.slug, content: stubMarkdown(step), ...origin(step) } } }]
    : step.entry ? [bullet(step)] : []));
  const out = new Map<ManagedEntity, EnrichmentResult | { error: unknown }>();
  for (const step of steps) {
    const reply = first.get(step);
    if (reply && 'error' in reply) { out.set(step, reply); continue; }
    const slug = step.result.slug;
    if (step.stub !== null) {
      try { await coordinatedDatabaseWrite(ctx, 'extract_entities', slug, [slug], tx => lowerDerivedPageIn(tx, step.derivation, step.sourceId, slug), step.derivation.trust); }
      catch (error) { out.set(step, { error }); }
    }
  }
  const second = await submit(steps.filter(step => step.stub !== null && step.entry && !step.inline && !out.has(step)).map(bullet));
  return Promise.all(steps.map(async step => {
    const reply = second.get(step) ?? (step.stub === null || step.inline ? first.get(step) : undefined);
    if (out.has(step)) return out.get(step)!;
    if (reply && 'error' in reply) return reply;
    const slug = step.result.slug;
    try {
      const response = reply?.response;
      // The door stamps agent_written or external_untrusted; an unknown-tier derivation lowers its new row as on an unmanaged brain.
      const stored = !response ? undefined : step.inline ? renderTimelineEntry(step.entry!, slug)!.canonical
        : response.status === 'ok' ? response.entry as { date: string; summary: string; source: string } | undefined : undefined;
      if (stored && step.derivation.trust.tier === 'unknown') await coordinatedDatabaseWrite(ctx, 'extract_entities', slug, [slug], async tx => {
        const rows = await tx.executeRaw<{ id: number }>(`SELECT te.id FROM timeline_entries te JOIN pages p ON p.id=te.page_id
          WHERE p.source_id=$1 AND p.slug=$2 AND te.date=$3::date AND te.summary=$4 AND COALESCE(te.source,'')=$5`, [step.sourceId, slug, stored.date, stored.summary, stored.source]);
        await lowerToDerivedTier(tx, 'timeline_entries', rows.map(row => Number(row.id)), step.derivation.trust);
      }, step.derivation.trust);
      const backlinkCreated = (await engine.executeRaw(`SELECT 1 FROM links l JOIN pages f ON f.id=l.from_page_id JOIN pages t ON t.id=l.to_page_id
        WHERE f.source_id=$1 AND f.slug=$2 AND t.source_id=$1 AND t.slug=$3 LIMIT 1`, [step.sourceId, slug, step.sourceSlug])).length > 0;
      return { ...step.result, timelineAdded: step.entry !== null, backlinkCreated };
    } catch (error) { return { error }; }
  }));
}

/**
 * Extract entities from text, then enrich each.
 * Uses simple regex patterns for entity detection.
 * This is the first fail-improve integration candidate (per Codex review).
 */
export async function extractAndEnrich(
  engine: BrainEngine,
  text: string,
  sourceSlug: string,
  opts?: EnrichmentTrustOptions & { throttle?: boolean; maxEntities?: number },
): Promise<EnrichmentResult[]> {
  // Bounded by default (#160 hardening): the greedy regex on a large paste
  // can produce thousands of hits; each enrichment is several DB round-trips.
  const entities = extractEntities(text).slice(0, opts?.maxEntities ?? 200);
  if (entities.length === 0) return [];

  const requests: EnrichmentRequest[] = entities.map(e => ({
    entityName: e.name,
    entityType: e.type,
    context: e.context,
    sourceSlug,
  }));

  return enrichEntities(engine, requests, { trusted: opts?.trusted, sourceId: opts?.sourceId, throttle: opts?.throttle });
}

// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------

/** Count entity mentions across the brain using keyword search. */
async function countMentions(
  engine: BrainEngine,
  entityName: string,
  sourceId?: string,
  excludePrivate = false,
): Promise<{ mentionCount: number; mentionSources: string[] }> {
  try {
    const results = await engine.searchKeyword(entityName, { limit: 100, ...(sourceId ? { sourceId } : {}), ...(excludePrivate ? { excludePrivate } : {}) });
    // Derive sources from slug prefixes since SearchResult has no metadata.skill
    const sources = new Set<string>();
    for (const r of results) {
      const prefix = r.slug.split('/')[0];
      if (prefix === 'people' || prefix === 'companies') sources.add('enrich');
      else if (prefix === 'meetings') sources.add('meeting-ingestion');
      else if (prefix === 'media') sources.add('media-ingest');
      else if (prefix === 'sources' || prefix === 'ideas') sources.add('idea-ingest');
      else if (prefix === 'voice-notes') sources.add('voice-note');
      else sources.add('brain-ops');
    }
    return { mentionCount: results.length, mentionSources: [...sources] };
  } catch {
    return { mentionCount: 0, mentionSources: [] };
  }
}

/** Suggest enrichment tier based on mention frequency. */
function suggestTier(
  mentionCount: number,
  mentionSources: string[],
  context: string,
): 1 | 2 | 3 {
  // 8+ mentions OR meeting/conversation source → Tier 1
  if (mentionCount >= 8) return 1;
  if (mentionSources.includes('meeting-ingestion') || mentionSources.includes('voice-note')) return 1;

  // 3-7 mentions across 2+ sources → Tier 2
  if (mentionCount >= 3 && mentionSources.length >= 2) return 2;

  // Default → Tier 3
  return 3;
}

/** Generate stub content for a new entity page. */
function generateStubContent(name: string, type: 'person' | 'company', context: string): string {
  if (type === 'person') {
    return `# ${name}\n\n**Type:** Person\n\n## Summary\n\n*Stub page. ${context}*\n\n## Timeline\n`;
  }
  return `# ${name}\n\n**Type:** Company\n\n## Summary\n\n*Stub page. ${context}*\n\n## Timeline\n`;
}

/** Simple entity extraction from text using regex patterns. */
export function extractEntities(text: string): Array<{ name: string; type: 'person' | 'company'; context: string }> {
  const entities: Array<{ name: string; type: 'person' | 'company'; context: string }> = [];
  const seen = new Set<string>();

  // Match capitalized multi-word names (likely people or companies)
  // Pattern: 2-4 capitalized words in sequence, scoped to a single line so a
  // paragraph break (or any line break) can't splice unrelated capitalized
  // words from adjacent sentences into one bogus candidate. The pattern
  // excludes all vertical whitespace -- CR, LF, VT (U+000B), FF (U+000C),
  // and the Unicode line/paragraph separators (U+2028/U+2029) -- while
  // retaining horizontal whitespace (spaces, tabs, NBSP, other Unicode
  // space separators), unlike a plain `[ \t]+`.
  const namePattern = /(?<!\p{L})(\p{Lu}[\p{Ll}\p{M}]+(?:[^\S\r\n\v\f\u2028\u2029]+\p{Lu}[\p{Ll}\p{M}]+){1,3})(?!\p{L})/gu;
  let match;
  while ((match = namePattern.exec(text)) !== null) {
    const name = match[1];
    if (seen.has(name.toLowerCase())) continue;
    seen.add(name.toLowerCase());

    // Simple heuristics for type classification
    const isCompany = /Inc\b|Corp\b|Ltd\b|LLC\b|Co\b|Labs?\b|Tech\b|AI\b|Capital\b|Ventures?\b|Fund\b/i.test(name);
    const type = isCompany ? 'company' : 'person';

    // Extract surrounding context (50 chars each side)
    const idx = match.index;
    const start = Math.max(0, idx - 50);
    const end = Math.min(text.length, idx + name.length + 50);
    const context = text.slice(start, end).replace(/\n/g, ' ').trim();

    entities.push({ name, type, context });
  }

  return entities;
}