Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.9 KB
Raw Blame
/**
 * Entity-extraction quarantine-lane operation cluster — pure move from
 * operations.ts (v0.46.x tranche 3): the issue-#160 extract_entities /
 * extraction_pending / extraction_review lane plus its cluster-local
 * resource-guard caps. Op consts stay module-private; `extractionOperations`
 * below lists them in EXACTLY the order they appear in the canonical
 * `operations` array in ../operations.ts. Never import from
 * '../operations.ts' here (cycle).
 */

import type { Operation } from './contract.ts';
import { opError, OperationError, type OperationContext } from './contract.ts';
import type { Action } from '../agent-output.ts';
import { hostOnlyError, paramUse, readFix } from './op-fix.ts';
import { sourceScopeOpts } from './context.ts';
import { randomUUID } from 'node:crypto';
import { unverifiedExtractionFragment, isUnverifiedExtraction, EXTRACTION_STATUS_KEY, STATUS_VERIFIED } from '../extraction-review.ts';
import { buildVisibilityClause } from '../search/sql-ranking.ts';
import { serializePageToMarkdown } from '../markdown.ts';
import { withoutBody } from '../page-body/page-shape.ts';
import type { Page } from '../types.ts';
import { submitPageMutation } from '../persistence/page-mutations.ts';
import { withPublicationGuard } from '../persistence/publication-guard.ts';
import { remoteTargetHidden } from '../persistence/page-visibility.ts';

// ---------------------------------------------------------------------------
// Extraction quarantine lane (issue #160)
//
// `extractAndEnrich` regex-extracts entity names from arbitrary text and
// creates people/ + companies/ stub pages. These three ops are its ONLY
// sanctioned surface:
//   - extract_entities    — run extraction. Direct authoritative writes need
//                           BOTH the trusted local CLI (ctx.remote === false)
//                           AND the explicit --trusted-extraction flag;
//                           everything else lands in the quarantine lane
//                           (frontmatter provenance/status markers).
//   - extraction_pending  — list unverified stubs awaiting review.
//   - extraction_review   — promote (status → verified) or reject
//                           (soft-delete) in batch. Owner-only (fail-closed
//                           on ctx.remote): THIS surface never lets a remote
//                           caller flip the status markers. Scope note: the
//                           markers are ordinary frontmatter, so a caller who
//                           already holds generic remote put_page write scope
//                           can rewrite the page (markers included) — that
//                           caller could equally author an unmarked people/
//                           page directly, so the lane adds no privilege
//                           there; put_page authz is its own boundary.
// ---------------------------------------------------------------------------

// Resource guards for extract_entities (#160 hardening): bound the work a
// single remote write-scope call can trigger. ponytail: flat caps; make them
// config knobs only if a real workload hits them.
const MAX_EXTRACT_TEXT_CHARS = 200_000;
const MAX_EXTRACT_ENTITIES = 200;

const extract_entities: Operation = {
  name: 'extract_entities',
  idempotent: false,
  outputRedaction: 'retrieval',
  description: 'Extract entity names (people, companies) from text and create/update their brain stub pages. Stubs from untrusted input land in the quarantine lane (frontmatter `provenance: auto-extracted` + `status: unverified`) — excluded from authoritative retrieval boosts until reviewed. Direct authoritative writes require the trusted local CLI with trusted_extraction set.',
  params: {
    text: { type: 'string', required: true, description: 'The text to extract entities from (email, transcript, pasted content, …). Max 200k characters — split larger inputs.' },
    source_slug: { type: 'string', required: true, description: 'Slug of the source page the text came from (used for backlinks + timeline attribution).' },
    trusted_extraction: { type: 'boolean', required: false, description: 'Local CLI only: write stubs directly as authoritative pages, skipping the quarantine lane. Ignored (always quarantined) for remote callers.' },
  },
  mutating: true,
  scope: 'write',
  handler: async (ctx, p) => {
    // Trust rule (#160, fail-closed like the CV6 provenance gate above):
    // `ctx.remote === false` is the ONLY truthy condition that can admit a
    // direct authoritative write, and even then the caller must opt in
    // explicitly. Remote/unset trust → quarantine lane, flag ignored.
    const trusted = ctx.remote === false && p.trusted_extraction === true;
    const text = p.text as string;
    // Resource guards: the greedy name regex on a huge paste can yield tens
    // of thousands of "entities", each costing several DB round-trips. Cap
    // input size loudly and entity count softly (surfaced as `truncated`).
    if (text.length > MAX_EXTRACT_TEXT_CHARS) {
      throw opError(
        'invalid_params',
        `extract_entities: text is ${text.length} chars (max ${MAX_EXTRACT_TEXT_CHARS}).`,
        `Nothing was written. Split the input into pieces of at most ${MAX_EXTRACT_TEXT_CHARS} characters and call extract_entities once per piece.`,
      );
    }
    if (ctx.dryRun) return { dry_run: true, action: 'extract_entities', trusted };
    const { extractEntities, enrichEntities } = await import('../enrichment-service.ts');
    const found = extractEntities(text);
    const capped = found.slice(0, MAX_EXTRACT_ENTITIES);
    // GBRA-431: a source page the remote caller cannot read is treated exactly as a missing one (no backlink to it).
    const sourceHidden = await remoteTargetHidden(ctx.engine, ctx.remote, ctx.sourceId || 'default', p.source_slug as string);
    const results = await enrichEntities(
      ctx.engine,
      capped.map((e) => ({ entityName: e.name, entityType: e.type, context: e.context, sourceSlug: p.source_slug as string })),
      {
        trusted,
        ...(ctx.sourceId ? { sourceId: ctx.sourceId } : {}),
        ctx,
        sourceHidden,
        // Pure local DB writes — no external API call to pace, so the
        // system-load capacity gate would only stall the caller.
        throttle: false,
      },
    );
    return {
      status: 'ok',
      trusted,
      quarantined: results.filter((r) => r.quarantined === true).length,
      count: results.length,
      entities_found: found.length,
      truncated: found.length > capped.length,
      entities: results,
    };
  },
  cliHints: { name: 'extract-entities' },
};

const extraction_pending: Operation = {
  name: 'extraction_pending',
  mutating: false,
  idempotent: true,
  outputRedaction: 'retrieval',
  description: 'List what waits for review: unverified auto-extracted entity stubs (pending; promote or reject them with extraction_review), facts saved without an entity (unattributed; relink each with remember entity + replaces, never forget first), and, when include names it, recently created entity pages whose name is one edit or one acronym letter from another entity (possible_duplicates; flag only, never merged).',
  params: {
    limit: { type: 'number', required: false, description: 'Max rows per list (default 100 for pending, 50 for the others; cap 500).' },
    offset: { type: 'number', required: false, description: 'Pagination offset for pending.' },
    include: { type: 'array', items: { type: 'string' }, required: false, description: 'Lists to return: pending, unattributed, possible_duplicates. Default pending and unattributed.' },
    unattributed_offset: { type: 'number', required: false, description: 'Pagination offset for unattributed (its next_offset).' },
    possible_duplicates_cursor: { type: 'string', required: false, description: 'Continuation for possible_duplicates (its next_cursor).' },
    since_days: { type: 'number', required: false, description: 'possible_duplicates: entity pages created in the last N days (default 30).' },
  },
  scope: 'read',
  handler: async (ctx, p) => {
    const include = parseInclude(p.include);
    const limit = Math.min(Math.max(Number(p.limit ?? 100) || 100, 1), 500);
    const offset = Math.max(Number(p.offset ?? 0) || 0, 0);
    // Read-side source isolation: route through sourceScopeOpts (federated
    // array > scalar > nothing), applied in SQL below.
    const scope = sourceScopeOpts(ctx);
    const params: unknown[] = [];
    let srcClause = '';
    if (scope.sourceIds && scope.sourceIds.length > 0) {
      params.push(scope.sourceIds);
      srcClause = `AND p.source_id = ANY($${params.length}::text[])`;
    } else if (scope.sourceId) {
      params.push(scope.sourceId);
      srcClause = `AND p.source_id = $${params.length}`;
    }
    params.push(limit, offset);
    const rows = await ctx.engine.executeRaw<{
      slug: string; title: string; type: string; source_id: string;
      extracted_from: string | null; created_at: string;
    }>(
      `SELECT p.slug, p.title, p.type, p.source_id,
              p.frontmatter ->> 'source' AS extracted_from,
              p.created_at::text AS created_at
       FROM pages p
       JOIN sources s ON s.id = p.source_id
       WHERE ${unverifiedExtractionFragment('p')}
         ${buildVisibilityClause('p', 's')}
         ${srcClause}
       ORDER BY p.created_at DESC
       LIMIT $${params.length - 1} OFFSET $${params.length}`,
      params,
    );
    const out: Record<string, unknown> = { count: rows.length, pending: include.has('pending') ? rows : [], include: [...include] };
    const lists = await import('../review-lists.ts');
    const listLimit = Math.min(Math.max(Number(p.limit ?? lists.UNATTRIBUTED_DEFAULT_LIMIT) || lists.UNATTRIBUTED_DEFAULT_LIMIT, 1), lists.REVIEW_LIST_MAX_LIMIT);
    const sourceIds = scope.sourceIds?.length ? scope.sourceIds : [scope.sourceId ?? ctx.sourceId ?? 'default'];
    if (include.has('unattributed')) {
      out.unattributed = await lists.listUnattributedFacts(ctx.engine, ctx, sourceIds,
        { limit: listLimit, offset: Math.max(Number(p.unattributed_offset ?? 0) || 0, 0) });
    }
    if (include.has('possible_duplicates')) {
      const { resolveExcludePrivatePages } = await import('../search/private-visibility.ts');
      out.possible_duplicates = await lists.listPossibleDuplicates(ctx.engine, ctx, sourceIds[0]!, {
        limit: listLimit, cursor: p.possible_duplicates_cursor,
        sinceDays: Math.max(Number(p.since_days ?? lists.POSSIBLE_DUPLICATES_DEFAULT_DAYS) || lists.POSSIBLE_DUPLICATES_DEFAULT_DAYS, 1),
        excludePrivate: await resolveExcludePrivatePages(ctx.engine, ctx.remote),
      });
    }
    return out;
  },
  cliHints: { name: 'extraction-pending' },
};

const PENDING_LISTS = ['pending', 'unattributed', 'possible_duplicates'] as const;

function parseInclude(raw: unknown): Set<string> {
  const given = Array.isArray(raw) ? raw : typeof raw === 'string' ? raw.split(',') : null;
  if (!given) return new Set(['pending', 'unattributed']);
  const wanted = new Set(given.map(v => String(v).trim()).filter(Boolean));
  const unknown = [...wanted].filter(v => !(PENDING_LISTS as readonly string[]).includes(v));
  if (unknown.length) {
    throw opError('invalid_params', `extraction_pending: unknown include ${unknown.map(v => JSON.stringify(v)).join(', ')}.`,
      `Nothing was read. Pass include as any of ${PENDING_LISTS.join(', ')}.`);
  }
  return wanted;
}

const REVIEW_SLUG = /^[a-z0-9][a-z0-9/_.-]{0,254}$/i;

function pendingFix(): Action {
  return readFix('Lists the unverified auto-extracted stubs awaiting review, with their slugs.',
    { argv: ['gbrain', 'extraction-pending'], mcp: { tool: 'extraction_pending', arguments: {} } });
}

const REVIEW_ACTIONS = ['promote', 'reject', 'distinct'] as const;
type ReviewDecision = typeof REVIEW_ACTIONS[number];
const isReviewAction = (v: unknown): v is ReviewDecision => typeof v === 'string' && (REVIEW_ACTIONS as readonly string[]).includes(v);

function reviewOwnerOnly(ctx: OperationContext, p: Record<string, unknown>) {
  const message = 'extraction_review is owner-only: promote, reject and distinct decisions must come from the trusted local CLI.';
  const action = isReviewAction(p.action) ? p.action : undefined;
  const slugs = Array.isArray(p.slugs) ? p.slugs : [];
  const of = typeof p.of === 'string' && REVIEW_SLUG.test(p.of) ? p.of : undefined;
  const exact = action && slugs.length > 0 && slugs.every(s => typeof s === 'string' && REVIEW_SLUG.test(s)) && (action !== 'distinct' || of);
  return exact
    ? hostOnlyError(ctx, 'permission_denied', message, ['gbrain', 'extraction-review', action, '--slugs', slugs.join(','), ...(action === 'distinct' ? ['--of', of!] : [])],
      'The review decision is the quarantine trust gate, so only the brain owner\'s CLI may make it.',
      { consent: action === 'reject' ? ['destructive'] : [] })
    : hostOnlyError(ctx, 'permission_denied', message, ['gbrain', 'extraction-pending'],
      'The review decision is the quarantine trust gate; the owner reviews the pending stubs and decides on the host.');
}

const extraction_review: Operation = {
  name: 'extraction_review',
  idempotent: false,
  outputRedaction: 'no_stored_text',
  description: 'Decide on review rows (batch). promote flips an unverified auto-extracted stub\'s `status` to verified (provenance kept for audit); reject soft-deletes the stub; distinct records that each of slugs is a different entity from of (`distinct_from` frontmatter), so possible_duplicates stops pairing them. Every decision writes through the page door: Git first on a source with a repository, the database on a repo-less source. Owner-only: this op is refused for any non-local caller. (The markers are ordinary frontmatter — the boundary against rewriting them wholesale is put_page write authz, same as for any page.)',
  params: {
    action: { type: 'string', required: true, description: "'promote', 'reject' or 'distinct'." },
    slugs: { type: 'array', required: true, items: { type: 'string' }, description: 'Slugs to act on (batch).' },
    of: { type: 'string', required: false, description: "distinct only: the existing entity page each of slugs is not the same as." },
  },
  mutating: true,
  scope: 'write',
  localOnly: true, cliOnly: { argv: ['gbrain', 'extraction-review', '<action>', '--slugs', '<slugs>'] },
  handler: async (ctx, p) => {
    // The review decision IS the trust gate — if a remote caller could
    // promote, injected content could self-promote and the quarantine lane
    // would be decorative. Fail-closed: only strictly-local callers pass.
    if (ctx.remote !== false) {
      throw reviewOwnerOnly(ctx, p);
    }
    const action = p.action;
    if (!isReviewAction(action)) {
      throw opError('invalid_params', `extraction_review: action must be 'promote', 'reject' or 'distinct'; got '${String(action)}'.`,
        "Nothing changed. Pass 'promote', 'reject' or 'distinct' as the action; fix lists the rows awaiting review.", { fix: pendingFix() });
    }
    // CLI passes `--slugs a,b,c` as one string; MCP passes a real array.
    const slugs = Array.isArray(p.slugs)
      ? (p.slugs as string[])
      : typeof p.slugs === 'string'
        ? p.slugs.split(',').map((s) => s.trim()).filter(Boolean)
        : [];
    if (slugs.length === 0) {
      throw opError('invalid_params', 'extraction_review: slugs must be a non-empty array.',
        `Nothing changed. Pass the stub slugs to act on, for example ${paramUse(ctx, 'slugs', ctx.remote === false ? 'slug1,slug2' : ['slug1', 'slug2'])}; fix lists the stubs awaiting review.`,
        { fix: pendingFix() });
    }
    const of = typeof p.of === 'string' ? p.of.trim() : '';
    if (action === 'distinct' && !REVIEW_SLUG.test(of)) {
      throw opError('invalid_params', 'extraction_review distinct needs of: the existing entity page the slugs are not the same as.',
        'Nothing changed. Pass of: the existing entity page the slugs are not the same as; the possible_duplicates rows of extraction_pending carry the exact command.',
        { fix: readFix('Lists recently created entity pages flagged as near names, each with its distinct command.',
          { argv: ['gbrain', 'extraction-pending', '--include', 'possible_duplicates'], mcp: { tool: 'extraction_pending', arguments: { include: ['possible_duplicates'] } } }) });
    }
    if (ctx.dryRun) return { dry_run: true, action: `extraction_review:${action}`, slugs, ...(action === 'distinct' ? { of } : {}) };
    const results: Array<{ slug: string; status: string; error?: ReturnType<OperationError['toJSON']> }> = [];
    const writeRequests: Array<{ slug: string; request_id: unknown; state: unknown }> = [];
    const failures: OperationError[] = [];
    for (const slug of slugs) {
      // First-match read is safe here: every write below keys on the RETURNED
      // row's page.source_id, so read and write can never target different
      // rows. gbrain-allow-unscoped-getpage: write follows the returned row
      const page = await ctx.engine.getPage(slug, ctx.sourceId ? { sourceId: ctx.sourceId } : undefined);
      if (!page) {
        results.push({ slug, status: 'not_found' });
        continue;
      }
      // A decision that fails is reported for its slug and the batch goes on: the decisions before it already landed.
      const decided = await reviewDecision(ctx, action, slug, page.source_id, of).catch((error: unknown) => {
        if (!(error instanceof OperationError)) throw error;
        failures.push(error);
        return null;
      });
      if (!decided) {
        results.push({ slug, status: 'failed', error: failures.at(-1)!.toJSON() });
        continue;
      }
      results.push({ slug, status: decided.status });
      if (decided.response) writeRequests.push({ slug, request_id: decided.response.request_id, state: decided.response.state });
    }
    if (failures.length === slugs.length) throw failures[0];
    return { status: failures.length ? 'partial' : 'ok', action, results, ...(writeRequests.length ? { write_requests: writeRequests } : {}) };
  },
  cliHints: { name: 'extraction-review', positional: ['action'] },
};

/**
 * One review decision under the source's publication guard: re-read the page, re-check that the decision
 * still applies, and publish it through the page door with the revision it read (a page that moved in
 * between reports `changed`; run the command again).
 */
async function reviewDecision(ctx: OperationContext, action: ReviewDecision, slug: string, sourceId: string, of: string):
  Promise<{ status: string; response?: Record<string, unknown> }> {
  return withPublicationGuard(ctx.engine, sourceId, async () => {
    const snapshot = await ctx.engine.readPageSnapshot(slug, { sourceId });
    if (!snapshot || snapshot.page.deleted_at) return { status: 'not_found' };
    const fm = (snapshot.page.frontmatter ?? {}) as Record<string, unknown>;
    // Frontmatter-only decisions edit the page's head (serialized with an empty body, as edit_page lays it out), so
    // the body is never read or rewritten and a segmented page keeps its segments.
    const head = (frontmatter: Record<string, unknown>) =>
      serializePageToMarkdown({ ...withoutBody(snapshot.page), frontmatter, compiled_truth: '', timeline: '' } as Page, snapshot.tags).slice(0, -1);
    const editHead = (frontmatter: Record<string, unknown>) =>
      ({ operation: 'edit_page' as const, params: { slug, source_id: sourceId, edits: [{ old_text: head(fm), new_text: head(frontmatter) }] } });
    let mutation: { operation: 'edit_page' | 'delete_page'; params: Record<string, unknown> };
    let status: string;
    if (action === 'distinct') {
      if (of === slug) return { status: 'same_page' };
      const other = await ctx.engine.getPage(of, { sourceId });
      if (!other) return { status: 'of_not_found' };
      const prior = Array.isArray(fm.distinct_from) ? fm.distinct_from.filter((x): x is string => typeof x === 'string') : typeof fm.distinct_from === 'string' ? [fm.distinct_from] : [];
      if (prior.includes(of)) return { status: 'already_distinct' };
      mutation = editHead({ ...fm, distinct_from: [...prior, of] });
      status = 'distinct';
    } else if (!isUnverifiedExtraction(fm)) {
      return { status: 'not_unverified' };
    } else if (action === 'promote') {
      // provenance stays 'auto-extracted' as the audit trail of HOW the page came to exist; status → 'verified'
      // records the owner's call.
      mutation = editHead({ ...fm, [EXTRACTION_STATUS_KEY]: STATUS_VERIFIED });
      status = 'promoted';
    } else {
      mutation = { operation: 'delete_page', params: { slug, source_id: sourceId } };
      status = 'rejected';
    }
    try {
      const response = await submitPageMutation(ctx, { operation: mutation.operation,
        params: { ...mutation.params, expected_revision: snapshot.revision, request_id: randomUUID() } });
      return { status, response };
    } catch (error) {
      if (error instanceof OperationError && ['revision_conflict', 'changed_since_preview'].includes(error.code)) return { status: 'changed' };
      throw error;
    }
  });
}

export const extractionOperations: Operation[] = [
  extract_entities, extraction_pending, extraction_review,
];