Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.1 KB
Raw Blame
// src/core/extract-takes-from-pages.ts
// v0.41.18.0 (A12, A24, T9). Haiku classifier loop over allowlisted page
// types — concept, atom, lore, briefing, writing, originals — extracts
// gradeable claims and inserts them as takes fence rows.
//
// Two-gate consent per A12:
//   - takes.bootstrap_enabled (default false): must be true to run at all.
//     Even manual `gbrain takes extract --from-pages` refuses without it.
//   - takes.autopilot_allowed (default false): must be true for autopilot's
//     auto-apply tier to fire the takes-bootstrap remediation.
//
// A24 deliberately limits autopilot to manual_only until v0.42.1 lands a
// 100+-case eval suite. v0.42 ships the classifier + CLI; autopilot stays
// blocked until eval coverage catches up.

import { existsSync, readFileSync } from 'node:fs';
import { loadConfig } from './config.ts';
import type { BrainEngine } from './engine.ts';
import type { TakeKind } from './engine.ts';
import type { OperationContext } from './ops/contract.ts';
import { chat, getChatModel, isAvailable } from './ai/gateway.ts';
import {
  appendTakesToPageBody,
  appendTakesToPageMdFirst,
  materializeTakeResolutions,
  isSafeFenceCellText,
  resolveTakesRepoDir,
  resolveTakesWritePath,
  TakesWriteError,
  withStoredTakesFence,
} from './takes-write.ts';
import { getWorktreeBinding, managedPersistenceEnabled } from './persistence/ownership.ts';
import { isWriteThroughDisabled, resolvePageWriteTarget } from './write-through.ts';
import { isDbOnly, loadStorageConfig } from './storage-config.ts';
import { serializePageToMarkdown } from './markdown.ts';
import { BudgetMeter, loadPricingOverrides } from './cycle/budget-meter.ts';
import { parseTakesFence } from './takes-fence.ts';

export const ALLOWED_PAGE_TYPES = [
  'concept', 'atom', 'lore', 'briefing', 'writing', 'originals',
] as const;

const CLASSIFIER_SYSTEM = `You extract gradeable CLAIMS from longform writing.

Output strict JSON: an array of objects with shape:
  {"claim": "<short imperative or assertion, <= 200 chars>",
   "kind": "fact" | "take" | "bet" | "hunch",
   "weight": 0.0..1.0}

Kind taxonomy:
  - fact: verifiable as true/false (e.g. "X raised $5M in Mar 2024")
  - take: a stated opinion that could be wrong (e.g. "X is undervalued")
  - bet:  a forward-looking prediction (e.g. "X will IPO in 2026")
  - hunch: a low-confidence gut feeling (e.g. "Y feels overstretched")

Skip pure narrative, questions, definitions, or pure quotes from others.
Max 15 claims per page; output [] if no gradeable claims are present.`;

export interface ExtractTakesFromPagesOpts {
  /** Required: must be true for any work to happen (A12). */
  bootstrapEnabled: boolean;
  /**
   * Dry-run: write nothing. Without `yes` it only lists the candidate pages
   * and calls no model; with `yes` it also classifies them.
   */
  dryRun?: boolean;
  /** The user passed --yes: page content may be sent to the chat model. */
  yes?: boolean;
  /** Scope to a single source. */
  sourceIdFilter?: string;
  /** Max pages to classify per run (caps cost). Default 50. */
  maxPages?: number;
  /**
   * Resume point from an earlier run's `next_before` (#5043): only pages
   * ordered after it (older `updated_at`, or the same instant and a lower id)
   * are selected.
   */
  before?: { updatedAt: string; id: number };
  /**
   * Also rescan pages that already hold takes (refresh semantics).
   * Default false: bootstrap runs skip covered pages, so repeated runs
   * PROGRESS through a corpus larger than one run's cap instead of
   * rescanning the same most-recently-updated slice forever.
   */
  includeCovered?: boolean;
  /** Owner identifier for the inserted takes. Default 'system'. */
  holder?: string;
  /** Model override; defaults to facts.extraction_model. */
  model?: string;
  /**
   * USD cap for the run's classifier calls. Default: config
   * `takes.bootstrap_budget_usd`, else 5.0 (the propose_takes default).
   * 0 disables the cap.
   */
  budgetUsd?: number;
  /** Progress hook called per page. */
  onProgress?: (done: number, total: number, claims: number) => void;
}

export interface ExtractTakesFromPagesResult {
  pages_scanned: number;
  /** Null on a dry run without `yes`: no claims were classified. */
  claims_extracted: number | null;
  /** Set on a dry run without `yes`: candidates listed, nothing sent to the model. */
  dry_run?: true;
  /** The candidate pages a dry run without `yes` listed. */
  candidates?: Array<{ slug: string; source_id: string }>;
  /**
   * Where the next run should resume, as `<updated_at text>,<page id>`: the
   * position of the last page this run finished with, or null if it finished
   * none. Pages that yield no claims stay uncovered and would otherwise be
   * selected first forever (#5043); passing this to `--before` moves past
   * them. A page a budget stop never reached is not counted as finished, and
   * one skipped on an error stays uncovered for a run without `--before`.
   */
  next_before: string | null;
  /** True if the run was a no-op because bootstrapEnabled is false. */
  consent_gate_blocked: boolean;
  /** True if chat gateway is unavailable (no LLM call possible). */
  llm_unavailable: boolean;
  /**
   * #4473 — pages the md-first writer refused (takes are markdown-canonical;
   * a page with no locatable .md file is skipped BEFORE the LLM call, never
   * written DB-only). Skipped pages hold no takes, so future runs rescan them.
   */
  pages_skipped: number;
  skipped: Array<{ slug: string; reason: string }>;
  /** Count of pages whose md write landed but whose DB mirror warned (reconcile heals). */
  mirror_warnings: number;
  /** True when the run stopped because the next classifier call would exceed the USD budget. */
  budget_exhausted: boolean;
  /** Extracted claims dropped because the page's takes fence already holds them. */
  duplicates_skipped: number;
}

/** Duplicate key for a take claim: case, whitespace and trailing punctuation aside. */
function claimKey(claim: string, holder: string): string {
  return `${holder}\u0000${claim.toLowerCase().replace(/\s+/g, ' ').trim().replace(/[.!?;:,]+$/, '')}`;
}

async function resolveBudgetUsd(engine: BrainEngine, explicit: number | undefined): Promise<number> {
  if (typeof explicit === 'number' && Number.isFinite(explicit) && explicit >= 0) return explicit;
  const raw = await engine.getConfig('takes.bootstrap_budget_usd').catch(() => null);
  const parsed = raw == null ? NaN : Number.parseFloat(raw);
  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 5.0;
}

interface PageRow {
  id: number;
  slug: string;
  source_id: string;
  type: string;
  compiled_truth: string;
  updated_at: string | Date;
  /** `<updated_at::text>,<id>`, the value `next_before` reports for this page. */
  resume_at: string;
}

/** Whether an unmanaged page's database body is its copy of record: write-through off, or its target in a storage.db_only directory. */
async function databaseOnlyPage(engine: BrainEngine, slug: string, sourceId: string): Promise<boolean> {
  if (await isWriteThroughDisabled(engine)) return true;
  const target = await resolvePageWriteTarget(engine, slug, sourceId);
  if (!target.ok) return false;
  try {
    const storage = loadStorageConfig(target.writeRoot);
    return storage !== null && isDbOnly(slug, storage);
  } catch {
    return false;
  }
}

/**
 * Pure helper: parse Haiku JSON output into typed claims. Returns []
 * on any parse failure (caller treats as "no claims extracted").
 */
export function parseClaimsJson(raw: string): Array<{ claim: string; kind: TakeKind; weight: number }> {
  try {
    // Strip code fences if model wrapped output in ```json.
    let text = raw.trim();
    const fenceMatch = text.match(/^```(?:json)?\n?([\s\S]*?)\n?```$/);
    if (fenceMatch) text = fenceMatch[1].trim();
    const parsed = JSON.parse(text);
    if (!Array.isArray(parsed)) return [];
    const valid: Array<{ claim: string; kind: TakeKind; weight: number }> = [];
    for (const item of parsed) {
      if (!item || typeof item !== 'object') continue;
      const claim = typeof item.claim === 'string' ? item.claim.trim().slice(0, 200) : '';
      const kind = typeof item.kind === 'string' ? item.kind : '';
      const weightRaw = typeof item.weight === 'number' ? item.weight : 0.5;
      const weight = Math.max(0, Math.min(1, weightRaw));
      if (!claim || !['fact', 'take', 'bet', 'hunch'].includes(kind)) continue;
      valid.push({ claim, kind, weight });
    }
    return valid;
  } catch {
    return [];
  }
}

export async function extractTakesFromPages(
  engine: BrainEngine,
  opts: ExtractTakesFromPagesOpts,
): Promise<ExtractTakesFromPagesResult> {
  const emptyTail = { next_before: null, pages_skipped: 0, skipped: [], mirror_warnings: 0, budget_exhausted: false, duplicates_skipped: 0 };
  // A12 consent gate: refuse without bootstrap_enabled even on manual call.
  if (!opts.bootstrapEnabled) {
    return {
      pages_scanned: 0,
      claims_extracted: 0,
      consent_gate_blocked: true,
      llm_unavailable: false,
      ...emptyTail,
    };
  }

  const dryRun = opts.dryRun ?? false;
  const listOnly = dryRun && !opts.yes;
  if (!listOnly && !isAvailable('chat')) {
    return {
      pages_scanned: 0,
      claims_extracted: 0,
      consent_gate_blocked: false,
      llm_unavailable: true,
      ...emptyTail,
    };
  }

  const managedJournalWrites = await managedPersistenceEnabled(engine);
  const maxPages = opts.maxPages ?? 50;
  const holder = opts.holder ?? 'system';
  const params: unknown[] = [];
  const bind = (value: unknown): string => `$${params.push(value)}`;
  const scope: string[] = [];
  if (opts.sourceIdFilter) scope.push(`source_id = ${bind(opts.sourceIdFilter)}`);
  // Resume strictly after the caller's position in the (updated_at DESC,
  // id DESC) order. The timestamp is bound as text and cast in SQL so its
  // microseconds survive; a parameter typed as timestamptz goes through a JS
  // Date in postgres.js and loses them.
  if (opts.before) scope.push(`(updated_at, id) < (${bind(opts.before.updatedAt)}::text::timestamptz, ${bind(opts.before.id)})`);

  // Fetch eligible pages. Order by updated_at DESC so recently-edited
  // pages get bootstrapped first.
  const typesList = ALLOWED_PAGE_TYPES.map((t) => `'${t}'`).join(', ');
  // Bootstrap progression: skip pages that already hold takes (opt out via
  // includeCovered). Without this, the updated_at-DESC + LIMIT selection made
  // every re-run rescan the same most-recent slice — a corpus larger than one
  // run's cap could never be fully bootstrapped (and each rescan re-spent LLM
  // budget on covered pages for upsert-identical rows).
  const coveredFilter = opts.includeCovered
    ? ''
    : `AND NOT EXISTS (SELECT 1 FROM takes t WHERE t.page_id = pages.id)`;
  const pages = await engine.executeRaw<PageRow>(
    `SELECT id, slug, source_id, type, compiled_truth, updated_at, updated_at::text || ',' || id AS resume_at
       FROM pages
      WHERE type IN (${typesList})
        AND deleted_at IS NULL
        AND length(COALESCE(compiled_truth, '')) > 200
        ${coveredFilter}
        ${scope.map((condition) => `AND ${condition}`).join(' ')}
      ORDER BY updated_at DESC, id DESC
      LIMIT ${maxPages}`,
    params,
  );

  if (listOnly) {
    return {
      ...emptyTail,
      dry_run: true,
      pages_scanned: pages.length,
      claims_extracted: null,
      next_before: pages.at(-1)?.resume_at ?? null,
      consent_gate_blocked: false,
      llm_unavailable: false,
      candidates: pages.map((page) => ({ slug: page.slug, source_id: page.source_id })),
    };
  }

  let pagesScanned = 0;
  let claimsExtracted = 0;
  let pagesSkipped = 0;
  let mirrorWarnings = 0;
  let budgetExhausted = false;
  let duplicatesSkipped = 0;
  const skipped: Array<{ slug: string; reason: string }> = [];
  const model = opts.model || getChatModel();
  const meter = new BudgetMeter({
    budgetUsd: await resolveBudgetUsd(engine, opts.budgetUsd),
    phase: 'takes_bootstrap',
    pricingOverrides: await loadPricingOverrides(engine),
  });
  // #4473: takes are markdown-canonical (takes-write.ts contract), so the
  // bootstrap routes every write through the fence writer instead of minting
  // DB-only rows the next reconcile/extract would clobber.
  const repoDir = await resolveTakesRepoDir(engine);

  const skipPage = (slug: string, reason: string) => {
    pagesSkipped++;
    skipped.push({ slug, reason });
  };

  for (const page of pages) {
    pagesScanned++;
    opts.onProgress?.(pagesScanned, pages.length, claimsExtracted);

    if (!page.compiled_truth || page.compiled_truth.length < 200) continue;

    // The database is the page's copy of record on a managed brain, and on an
    // unmanaged one when write-through is off or the page sits in a declared
    // storage.db_only directory: its takes are composed from the database
    // snapshot and published through put_page, never from a file. A source
    // with a canonical owner (bound before the run, or by this run's first
    // put_page) fences the md-first writer, so its file-backed pages publish
    // through put_page too.
    const databaseOnly = !dryRun && await databaseOnlyPage(engine, page.slug, page.source_id);
    const ownedFileBacked = !dryRun && !managedJournalWrites && !databaseOnly && await getWorktreeBinding(engine, page.source_id) !== null;
    const fromDatabase = managedJournalWrites || databaseOnly || ownedFileBacked;

    // #4473: locate a file-backed page's markdown home BEFORE the LLM call — a
    // page the fence writer would refuse must not burn classifier budget
    // (skipped pages hold no takes, so every future run would re-classify them).
    let mdPath: string | null = null;
    if (!dryRun && (!fromDatabase || ownedFileBacked)) {
      try {
        const { path } = await resolveTakesWritePath(engine, repoDir, page.slug, page.source_id);
        if (existsSync(path)) mdPath = path;
      } catch {
        // mirror_unavailable (no repo dir + no source local_path)
      }
      if (!mdPath) {
        skipPage(page.slug, 'mirror_unavailable');
        continue;
      }
    }

    // Pin the selected page before spending time on extraction. A publication
    // with a newer revision must skip the whole page atomically.
    const managedSnapshot = fromDatabase && !dryRun
      ? await engine.readPageSnapshot(page.slug, { sourceId: page.source_id }) : null;
    if (fromDatabase && !dryRun && (!managedSnapshot || managedSnapshot.page.id !== page.id)) {
      skipPage(page.slug, 'page_identity_changed');
      continue;
    }

    // Truncate to keep per-page cost bounded (~20K chars → ~5K input tokens).
    const text = (managedSnapshot?.page.compiled_truth ?? page.compiled_truth).slice(0, 20_000);

    const budget = meter.check({
      modelId: model,
      estimatedInputTokens: Math.ceil((CLASSIFIER_SYSTEM.length + text.length) / 4) + 50,
      maxOutputTokens: 2000,
      label: 'takes_bootstrap',
    });
    if (!budget.allowed) {
      budgetExhausted = true;
      break;
    }

    let response: { text: string };
    try {
      response = await chat({
        // #2997 — default to the configured chat model (file-plane gateway
        // config, same idiom as enrich.ts) instead of hardcoded cloud Haiku.
        // On OAuth/local-only installs the hardcoded model made every takes
        // extraction die with llm_unavailable despite a working chat_model.
        model,
        system: CLASSIFIER_SYSTEM,
        messages: [
          {
            role: 'user',
            content: `<page slug="${page.slug}" type="${page.type}">\n${text}\n</page>`,
          },
        ],
        maxTokens: 2000,
      });
    } catch (err) {
      // Skip pages whose chat call fails (rate limit, content filter, auth,
      // transient error) with the reason, so an outage is distinguishable
      // from "nothing to extract". Per-page progress continues.
      const code = (err as { code?: unknown; status?: unknown } | null)?.code
        ?? (err as { status?: unknown } | null)?.status
        ?? (err instanceof Error ? err.name : 'unknown');
      skipPage(page.slug, `llm_error:${String(code)}`);
      continue;
    }

    const claims = parseClaimsJson(response.text);
    if (claims.length === 0) continue;

    if (dryRun) {
      claimsExtracted += claims.length;
      continue;
    }

    // #4473: md-first write through the fence pipeline (row numbers derive
    // from the fence — max existing + 1 — so the historical row_num=1
    // collision posture is gone). LLM output is pre-filtered against the
    // fence-cell guards so one garbled claim drops alone instead of sinking
    // the page's other claims.
    // Never append a claim the page's takes fence already holds (a rerun with
    // includeCovered, or a hand-added take): duplicate gradeable takes skew
    // calibration. The fence is canonical, so dedupe against its active rows.
    let duplicateBody: string;
    try {
      duplicateBody = await withStoredTakesFence(engine, page.id, fromDatabase
        ? serializePageToMarkdown(managedSnapshot!.page, managedSnapshot!.tags)
        : readFileSync(mdPath!, 'utf-8'));
    } catch (err) {
      if (!(err instanceof TakesWriteError)) throw err;
      skipPage(page.slug, err.code);
      continue;
    }
    const held = new Set(parseTakesFence(duplicateBody).takes
      .filter((t) => t.active).map((t) => claimKey(t.claim, t.holder)));
    const safeClaims = claims.filter((c) => {
      if (!isSafeFenceCellText(c.claim)) return false;
      const key = claimKey(c.claim, holder);
      if (held.has(key)) {
        duplicatesSkipped++;
        return false;
      }
      held.add(key);
      return true;
    });
    if (safeClaims.length === 0) continue;
    try {
      const rows = safeClaims.map((c) => ({
          claim: c.claim,
          kind: c.kind,
          holder,
          weight: c.weight,
          source: 'cli:takes-bootstrap-from-pages',
        }));
      if (fromDatabase) {
        const body = await materializeTakeResolutions(engine, managedSnapshot!.page.id, duplicateBody);
        const composed = appendTakesToPageBody(body, rows);
        const { operations } = await import('./operations.ts');
        const putPage = operations.filter(operation => !operation.localOnly).find(operation => operation.name === 'put_page');
        if (!putPage) throw new Error('put_page operation missing (gbrain build issue)');
        const config = loadConfig() ?? { engine: engine.kind };
        const ctx: OperationContext = {
          engine, config, logger: { info: () => {}, warn: () => {}, error: () => {} },
          dryRun: false, remote: false, sourceId: page.source_id,
        };
        await putPage.handler(ctx, {
          slug: page.slug,
          content: composed.body,
          expected_revision: managedSnapshot!.revision,
        });
        claimsExtracted += composed.rowNums.length;
      } else {
        const { rowNums, mirror } = await appendTakesToPageMdFirst(
          { engine, slug: page.slug, brainDir: repoDir, sourceId: page.source_id }, rows,
        );
        claimsExtracted += rowNums.length;
        if (mirror.mirror_warning) mirrorWarnings++;
      }
    } catch (err) {
      const code = (err as { code?: string; writeError?: string })?.code ?? (err as { writeError?: string })?.writeError;
      if (fromDatabase && (code === 'revision_conflict' || code === 'page_identity_changed' || code === 'source_changed'
        || (!managedJournalWrites && code === 'owner_unavailable'))) {
        skipPage(page.slug, code);
        continue;
      }
      if (err instanceof TakesWriteError) {
        // Skip + count (mirror_unavailable race, fence_unparsed, page_locked,
        // invalid_input) — never fall back to a DB-only write.
        skipPage(page.slug, err.code);
        continue;
      }
      throw err;
    }
  }

  // Every page the loop entered is finished except the one a budget stop broke on.
  const finished = budgetExhausted ? pagesScanned - 1 : pagesScanned;
  return {
    pages_scanned: pagesScanned,
    claims_extracted: claimsExtracted,
    next_before: finished > 0 ? pages[finished - 1]!.resume_at : null,
    consent_gate_blocked: false,
    llm_unavailable: false,
    pages_skipped: pagesSkipped,
    skipped,
    mirror_warnings: mirrorWarnings,
    budget_exhausted: budgetExhausted,
    duplicates_skipped: duplicatesSkipped,
  };
}