Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
60.1 KB
Raw Blame
/**
 * v0.36.1.0 (T3) — propose_takes cycle phase.
 *
 * Scans markdown pages updated since last run, sends each page's prose to
 * a tuned LLM extractor, writes the extracted gradeable claims to the
 * `take_proposals` queue. User accepts/rejects via `gbrain takes propose`.
 *
 * Idempotency contract (D17 schema spec):
 *   The unique index on (source_id, page_slug, content_hash, prompt_version)
 *   means an unchanged page never re-spends LLM tokens. Bumping
 *   PROPOSE_TAKES_PROMPT_VERSION cleanly invalidates the cache so a tuned
 *   prompt re-runs proposals on every page.
 *
 * F2 fence dedup:
 *   The phase reads the page's existing `<!-- gbrain:takes:begin -->` fence
 *   (when present) and passes the canonical take rows to the extractor as
 *   "things you have already captured." This prevents duplicate proposals
 *   when a user adds prose to a page that already has takes.
 *
 * Auto-resolve posture:
 *   propose_takes only WRITES proposals to the queue. Nothing here mutates
 *   the canonical takes table. Operator opt-in via `gbrain takes propose
 *   --accept N` is the only path from queue to canonical fence (D17).
 *
 * Prompt tuning status (v0.36.1.0 ship state):
 *   The default extractor prompt was tuned against the synthetic corpus at
 *   test/fixtures/calibration/ and validated via the cat15 propose_takes
 *   eval in the gbrain-evals repo. First live run scored 0.952 F1 on
 *   training (target 0.85) and 0.922 F1 on holdout (target 0.80), with a
 *   0.03 train-holdout gap (no overfitting). PROPOSE_TAKES_PROMPT_VERSION
 *   is "v0.36.1.0-tuned-cat15". Re-tuning requires re-running cat15;
 *   bumping the version string invalidates the take_proposals idempotency
 *   cache so old proposals stay as audit history but the next cycle
 *   re-extracts fresh against the new prompt.
 *
 * The extractor LLM call is INJECTED via opts.extractor for tests, so the
 * phase can run hermetically in unit tests without touching the gateway.
 */

import { observationDateLine, observationDateRule, resolveObservationDate, type ObservationDate } from '../ai/date-grounding.ts';
import { isConsumerDateGroundingOn } from '../facts/extract.ts';
import { randomUUID, createHash } from 'node:crypto';
import { BaseCyclePhase, CYCLE_DEADLINE_RESERVE_MS, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { defaultTimeoutMsFor } from '../minions/handler-timeouts.ts';
import {
  PROPOSE_TAKES_CALL_TIMEOUT_KEY, PROPOSE_TAKES_CALL_TIMEOUT_MAX_MS, PROPOSE_TAKES_CALL_TIMEOUT_MIN_MS, readPhaseConfigNumber,
} from './phase-config-values.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { probeChatModelWithKey } from '../ai/chat-key-probe.ts';
import { createGlobalLlmHaltTracker, haltedClassOf, type GlobalLlmErrorClass } from '../ai/errors.ts';
import { normalizeModelId } from '../model-id.ts';
import { writeReceipt } from '../extract/receipt-writer.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { upsertExtractRollup, classifyRunStop, providerHaltReason } from '../extract/rollup-writer.ts';
import { GBrainError } from '../types.ts';
import { isConfigTruthy } from '../config.ts';
import { matchingCloseBracket } from '../llm-json.ts';
import { TAKE_KIND_VALUES } from '../takes-fence.ts';
import { publicBody } from '../mentions/aliases.ts';
import { attributionChecksEnabled } from './attribution-checks.ts';
import { downgradeAssistantOnlyHolders } from './propose-takes-attribution.ts';
import { ProposeTakesFailures, PROPOSE_TAKES_HOLD_RUNS, type FailedPage } from './propose-takes-failures.ts';
import type { OperationContext } from '../operations.ts';
import type { BrainEngine } from '../engine.ts';
import type { PhaseStatus, CyclePhase } from '../cycle.ts';
import { guestSourceExclusionSql } from '../guests/derived.ts';

/**
 * Bump when the extractor prompt or the JSON output shape changes. Old
 * verdicts in `take_proposals` (composite key includes prompt_version) stay
 * valid as audit history; new runs re-spend LLM tokens on every page.
 */
export const PROPOSE_TAKES_PROMPT_VERSION = 'v0.36.1.0-tuned-cat15-kinds4736';

/**
 * Sentinel claim_text for the tombstone row written when a page extracts
 * ZERO gradeable claims. Without a tombstone the idempotency tuple is never
 * recorded, so every cycle re-spends an LLM call on unchanged zero-claim
 * prose — the "unchanged page never re-spends tokens" contract only held
 * for pages that produced >=1 claim. The tombstone is inserted with
 * status='rejected' so no pending-review query surfaces it as a live
 * proposal; its only job is to make the next cycle a cache hit.
 */
export const EMPTY_EXTRACTION_TOMBSTONE_TEXT = '(no gradeable claims)';

/**
 * Tuned extractor prompt, validated against the hand-labeled synthetic
 * corpus at test/fixtures/calibration/. Measured F1 on first live run
 * via gbrain-evals cat15 (claude-sonnet-4-6 extractor, claude-haiku-4-5
 * matcher judge):
 *
 *   training avg F1: 0.952 (target 0.85, exceeded by 10 points)
 *   holdout  avg F1: 0.922 (target 0.80, exceeded by 12 points)
 *   train-holdout gap: 0.03 (no overfitting signal)
 *
 * Per-genre F1 floor: 0.80 (people-pages, the hardest genre). The
 * concept-with-timeline and meeting-notes genres scored at 1.00 on
 * holdout pages.
 *
 * Design choices baked into the prompt:
 *   - Worked example list seeds the model's notion of "gradeable claim"
 *     so it doesn't drift into pure-fact extraction.
 *   - NOT-gradeable list catches the most common over-extraction modes
 *     (pure facts, direct quotes, restatements).
 *   - conviction inference rules anchored to specific hedging language
 *     ("I bet"/"strong conviction"=0.7-0.85, "I think"/"moderate"=0.5-0.7).
 *   - kind enum kept narrow — three tags; the v1 stub's 4-tag enum bled
 *     into noise classification. #4736: the tags now use the fence
 *     vocabulary parseExtractorOutput accepts ('take'|'bet'|'hunch'); the
 *     tuned prompt asked for prediction|judgment|bet, which the parser
 *     allowlist (fact|take|bet|hunch) coerced wholesale to 'take',
 *     destroying kind provenance on every extraction. Label-only change:
 *     what counts as gradeable is untouched, so the cat15 F1 numbers above
 *     still describe the extraction behavior. prediction/judgment stay
 *     mapped in the parser for cached/old-model outputs.
 *
 * Replaces the v0.36.1.0-stub. If you re-tune, run cat15 against the
 * fixtures before bumping PROPOSE_TAKES_PROMPT_VERSION; the train-holdout
 * gap should stay < 0.10 (overfitting threshold).
 */
export const EXTRACT_TAKES_PROMPT = `Extract gradeable claims from the prose below.

A "gradeable claim" is a prediction, recommendation, or interpretive judgment
that could turn out wrong over time. Examples:
- "X company will hit ARR milestone by Q3" (take: a prediction)
- "Y founder is going to struggle with execution" (take: a judgment)
- "Z market will compress in 18 months" (take: a prediction)
- "I bet alice wins the round" (bet)
- "Maybe DTC is quietly coming back" (hunch)

NOT gradeable (do NOT extract these):
- Pure facts ("X was founded in 2020")
- Direct quotes from others without endorsement
- Restatements of an earlier claim in the same page

For each gradeable claim, output a JSON object with:
- claim_text   (string, <=200 chars, paraphrase or near-verbatim from prose)
- kind         ('take' = prediction or interpretive judgment | 'bet' = explicit wager language | 'hunch' = low-conviction guess)
- holder       ('world' | 'people/<slug>' | 'companies/<slug>' | 'brain' — default 'brain' when author asserts the claim)
- weight       (number 0..1 inferred from hedging language: 'I bet'/'strong conviction'=0.7-0.85,
                'I think'/'moderate conviction'=0.5-0.7, 'maybe'/'I'd guess'=0.3-0.5)
- domain       (short tag — e.g. 'tactics', 'macro', 'hiring', 'geography', 'pricing')

Output ONLY a JSON array of these objects. No prose. No commentary. If no
gradeable claims, return [].

EXISTING FENCE ROWS (already captured — do NOT propose duplicates):
{EXISTING_TAKES_JSON}

PAGE PROSE:
{PAGE_BODY}
`;

/** One proposed take, as the extractor produces it. */
export interface ProposedTake {
  claim_text: string;
  kind: 'fact' | 'take' | 'bet' | 'hunch';
  holder: string;
  weight: number;
  domain?: string;
  /**
   * #4737: 'provider:modelId' of the model that ACTUALLY answered the
   * extraction call (ChatResult.model — alias/provider-recipe resolution
   * can differ from the configured string). Stamped by defaultExtractor;
   * optional so injected test extractors need not care. When present it
   * wins over the requested model for take_proposals.model_id provenance.
   */
  served_model?: string;
}

/**
 * #5425 (opt-in, `dream.propose_takes.attribution_rules=true`): speaker and
 * withdrawal rules for conversation pages, proposed by @clatyceo. Off by
 * default: a matched cat15-corpus run (Sonnet 4.6, 9 labeled pages x 3)
 * measured F1 0.896 → 0.876 (recall 0.924 → 0.882) with no attribution error
 * in either arm to fix. A separate prompt version keeps the two caches apart.
 */
export const EXTRACT_TAKES_ATTRIBUTION_RULES = `NOT gradeable either:
- Claims later withdrawn, corrected, or narrowed in the same page; omit them,
  or express only the final narrowed scope if it remains gradeable

Attribution: an assistant-authored gradeable judgment may use holder 'brain';
do not attribute it to the user or another person/company unless that speaker
explicitly endorses it. Assistant-added plans or deadlines are not user
commitments by default.

`;
export const PROPOSE_TAKES_ATTRIBUTION_PROMPT_SUFFIX = '+attribution5425';

/** Extractor function signature — injected for tests; production calls gateway. */
export type ProposeTakesExtractor = (input: {
  pagePath: string;
  pageBody: string;
  existingTakes: Array<{ claim: string; kind: string; holder: string; weight: number }>;
  modelHint?: string;
  /**
   * #4494: output cap for the extractor call (default
   * PROPOSE_TAKES_MAX_TOKENS). Configurable via dream.propose_takes.max_tokens
   * because thinking models spend reasoning tokens INSIDE maxTokens — at the
   * 2048 default a thinking model can burn the whole budget before emitting
   * any JSON, truncating EVERY page into a permanent per-page retry loop.
   */
  maxTokens?: number;
  /** #4494: escalated cap for the one truncation retry (default
   *  PROPOSE_TAKES_RETRY_MAX_TOKENS; clamped to >= maxTokens). */
  retryMaxTokens?: number;
  /** #5958: operator-set bound (ms) for each extractor call, base and retry
   *  alike. The phase reads dream.propose_takes.call_timeout_ms and caps it
   *  by the phase time left. Absent: extractorCallTimeoutMs(maxTokens). */
  callBoundMs?: number;
  /** #5425: include EXTRACT_TAKES_ATTRIBUTION_RULES (opt-in). */
  attributionRules?: boolean;
  /**
   * extraction.date_grounding: resolve relative deadlines ("by Q3", "in 18
   * months") against the page's observation date. Prompt-only; the
   * idempotency key (prompt_version) is unchanged so enabling it never
   * reprocesses pages.
   */
  dateGrounding?: boolean;
  /** The page's observation date (date-grounding.ts), null when undated. */
  observationDate?: ObservationDate | null;
}) => Promise<ProposedTake[]>;

export interface ProposeTakesOpts extends BasePhaseOpts {
  /** Brain repo root for fs-source page walking. Optional — defaults to engine pages. */
  repoPath?: string;
  /** Limit pages processed in this cycle (for triage / quick smoke). Default: 100. */
  pageLimit?: number;
  /** Inject the LLM call for tests; production uses gateway.chat. */
  extractor?: ProposeTakesExtractor;
  /** Override prompt_version (tests). */
  promptVersion?: string;
  /** Override model id (tests + config). */
  model?: string;
  /** Skip pages that already have a complete takes fence. Default: true. */
  skipPagesWithFence?: boolean;
  /** Override the phase wall-clock deadline (tests). Default: 30 min. */
  deadlineMs?: number;
  /**
   * #4102 — `gbrain dream --phase propose_takes --once` bypasses the
   * `cycle.propose_takes.enabled` off switch for THIS call only (mirrors the
   * conversation_facts_backfill `once` semantics; never reads/writes config).
   */
  once?: boolean;
}

export interface ProposeTakesResult {
  pages_scanned: number;
  cache_hits: number;
  cache_misses: number;
  proposals_inserted: number;
  /** Idempotency rows written for pages that extracted zero claims (or only claims rejected before, #7299). */
  tombstones_written: number;
  /** #7299: extracted claims dropped because an operator rejected the same claim text on the same page. */
  rejected_claims_skipped: number;
  budget_exhausted: boolean;
  /** True when the phase deadline fired before the page loop completed (partial result). */
  deadline_hit?: boolean;
  /**
   * Set when the page loop broke on a whole-run LLM failure (#3044):
   * auth/billing on the first hit, rate_limit after RATE_LIMIT_HALT_STREAK
   * consecutive hits. The phase reports 'warn' ('fail' when NO extractor
   * call succeeded) and the rollup records a halt so the condition can't
   * hide behind a green summary.
   */
  aborted_global_error?: GlobalLlmErrorClass;
  /**
   * #3763: set when the page loop halted because EVERY extractor call failed
   * (zero successes) for EXTRACTOR_FAILURE_HALT_STREAK consecutive pages —
   * a dead extractor lane (bad model id, broken recipe, systematic truncation)
   * that would otherwise re-bill every remaining page. Folds into `halted`
   * and reports the phase as 'fail'.
   */
  aborted_failure_streak?: boolean;
  /** Extractor calls that returned (idempotency cache hits don't count). */
  llm_calls_succeeded: number;
  /** Extractor calls that threw (global or per-page alike). */
  llm_calls_failed: number;
  /** GBRA-130 A-25: every page whose extractor call failed this run, with its counted attempts; a `held` page got a failure tombstone. */
  failed_pages: FailedPage[];
  /** Failure tombstones written this run (pages held after PROPOSE_TAKES_HOLD_RUNS counted runs failing the same way). */
  failure_tombstones_written: number;
  /** Set when most fresh pages failed (item_failures_systemic): this run's failures were recorded but not counted. */
  systemic?: { code: string; message: string };
  warnings: string[];
}

/** Narrow projection of `pages` — the only columns this phase reads. */
interface ProposeTakesPageRow {
  slug: string;
  source_id: string;
  compiled_truth: string | null;
}

/**
 * Load proposal candidates with a narrow projection instead of
 * `engine.listPages` (`SELECT p.*`). The phase only reads slug, source_id
 * and compiled_truth — skipping timeline/frontmatter/title keeps large
 * toasted columns out of the hot path. Scope precedence mirrors
 * `sourceScopeOpts`: federated array (`sourceIds`) beats scalar
 * (`sourceId`); ordering matches `PAGE_SORT_SQL.updated_desc` with an id
 * tiebreak for determinism. (Takeover of PR #1979's projection by
 * @shawnduggan.)
 */
async function listCandidatePages(
  engine: BrainEngine,
  scope: ScopedReadOpts,
  limit: number,
  types: string[] | null = null,
): Promise<ProposeTakesPageRow[]> {
  const where = [
    'deleted_at IS NULL',
    "type IS DISTINCT FROM 'extract_receipt'",
    // #5211: an atom is a fragment of a page that is itself a candidate; a legacy atom carries no writer stamp.
    "type IS DISTINCT FROM 'atom'",
    // #5212: never propose takes from the dream cycle's own output.
    "COALESCE(frontmatter->>'dream_generated', '') <> 'true'",
    "COALESCE(frontmatter->>'extracted_by', '') NOT LIKE 'extract_atoms%'",
    "COALESCE(frontmatter->>'synthesized_by', '') NOT LIKE 'synthesize_concepts%'",
    guestSourceExclusionSql('propose_takes', 'pages'),
  ];
  const params: unknown[] = [];
  if (scope.sourceIds && scope.sourceIds.length > 0) {
    params.push(scope.sourceIds);
    where.push(`source_id = ANY($${params.length}::text[])`);
  } else if (scope.sourceId) {
    params.push(scope.sourceId);
    where.push(`source_id = $${params.length}`);
  }
  // #5213: `cycle.propose_takes.types` narrows candidates to listed page types.
  if (types) {
    params.push(types);
    where.push(`type = ANY($${params.length}::text[])`);
  }
  params.push(limit);
  return engine.executeRaw<ProposeTakesPageRow>(
    `SELECT slug, source_id, compiled_truth
       FROM pages
      WHERE ${where.join(' AND ')}
      ORDER BY updated_at DESC, id DESC
      LIMIT $${params.length}`,
    params,
  );
}

export const PROPOSE_TAKES_TYPES_KEY = 'cycle.propose_takes.types';

/**
 * #5213: the page types propose_takes may scan, from `cycle.propose_takes.types`
 * (a JSON array of type names, the `cycle.enrich_thin.types` shape). Unset,
 * empty, unreadable or malformed means every type stays a candidate; a
 * malformed value also prints one stderr line so a typo is visible.
 */
export async function proposeTakesTypes(engine: BrainEngine): Promise<string[] | null> {
  const raw = await engine.getConfig?.(PROPOSE_TAKES_TYPES_KEY).catch(() => null) ?? null;
  if (raw == null || raw.trim() === '') return null;
  try {
    const parsed = JSON.parse(raw);
    if (Array.isArray(parsed)) {
      const types = parsed.filter((t): t is string => typeof t === 'string' && t.trim().length > 0).map(t => t.trim());
      if (types.length > 0) return types;
    }
  } catch {
    // reported below
  }
  process.stderr.write(`[propose_takes] ignoring ${PROPOSE_TAKES_TYPES_KEY}: expected a JSON array of page types, e.g. gbrain config set ${PROPOSE_TAKES_TYPES_KEY} '["meeting","note"]'\n`);
  return null;
}

/**
 * Compute the content_hash key for the idempotency cache. SHA-256 of the
 * page body suffices — page slug + prompt_version are separate columns in
 * the composite unique index.
 */
export function contentHash(pageBody: string): string {
  return createHash('sha256').update(pageBody).digest('hex');
}

/**
 * Detect whether a page already has a complete `<!-- gbrain:takes:begin -->`
 * fence. We DO propose against pages with fences (F2 dedup) but the operator
 * may opt to skip-with-fence pages via skipPagesWithFence:true for a faster
 * pass. The fence shape mirrors src/core/takes-fence.ts.
 */
export function hasCompleteFence(pageBody: string): boolean {
  return /<!---?\s*gbrain:takes:begin[\s\S]*?gbrain:takes:end\s*-->/.test(pageBody);
}

/**
 * Parse the existing fence into rows so the extractor can dedupe.
 * Returns [] when no fence is present. Best-effort — malformed fences
 * surface to the operator via the existing v0.28 fence parser, not here.
 */
export function extractExistingTakesForDedup(pageBody: string): Array<{
  claim: string;
  kind: string;
  holder: string;
  weight: number;
}> {
  const fenceMatch = pageBody.match(/<!---?\s*gbrain:takes:begin\s*-->([\s\S]*?)<!---?\s*gbrain:takes:end\s*-->/);
  if (!fenceMatch) return [];
  const body = fenceMatch[1] ?? '';
  const rows: Array<{ claim: string; kind: string; holder: string; weight: number }> = [];
  for (const line of body.split('\n')) {
    const cells = line.split('|').map(c => c.trim()).filter((_, i, arr) => i > 0 && i < arr.length - 1);
    // Skip header + separator rows.
    if (cells.length < 4) continue;
    if (cells[0] === '#' || cells[0]?.match(/^-+$/)) continue;
    const claim = cells[1] ?? '';
    if (!claim || claim.startsWith('~~')) continue; // strikethrough = inactive, doesn't count for dedup
    const kind = cells[2] ?? 'take';
    const holder = cells[3] ?? 'brain';
    const weight = Number.parseFloat(cells[4] ?? '0.5');
    rows.push({
      claim: claim.replace(/^~~|~~$/g, ''),
      kind,
      holder,
      weight: Number.isFinite(weight) ? weight : 0.5,
    });
  }
  return rows;
}

/** Per-call wall-clock timeout for the extractor LLM call at the base cap. */
const EXTRACTOR_CALL_TIMEOUT_MS = 90_000;
/** Ceiling for the scaled timeout: the gateway's own chat default
 *  (GBRAIN_AI_CHAT_TIMEOUT_MS, 300s) bounds the call regardless. */
const EXTRACTOR_CALL_TIMEOUT_MAX_MS = 300_000;

/**
 * #3763 — output caps for the extractor call. A stopReason 'length' response
 * at the base cap retries ONCE at the escalated cap (facts/extract.ts #2113
 * parity); a still-truncated retry throws an error NAMING the truncation
 * instead of the old generic 'transient — retry' (which re-billed the page
 * every cycle forever while hiding the real cause).
 *
 * #4494 — these are now DEFAULTS, overridable via
 * `dream.propose_takes.max_tokens` / `dream.propose_takes.retry_max_tokens`
 * (floor 256; retry clamped >= base), mirroring dream.triage.max_tokens.
 * Thinking models (DeepSeek-R1, MiniMax-M3, Claude with extended thinking)
 * spend reasoning tokens INSIDE the maxTokens budget, so field deployments
 * saw every dense page truncate at 2048 → retry at 4096 → truncate again →
 * throw → re-bill next cycle, forever. Raising the config key breaks that
 * loop without inflating the default for non-thinking models.
 */
export const PROPOSE_TAKES_MAX_TOKENS = 2048;
export const PROPOSE_TAKES_RETRY_MAX_TOKENS = 4096;

/**
 * Wall-clock timeout for one extractor call, scaled with its output cap
 * (#5771). A flat 90s sized for the 2048-token base call starved the
 * truncation retry: at a configured retry_max_tokens of 12000 a dense page's
 * retry needs well over 90s to generate, so it timed out every cycle and the
 * page was re-billed (base call + aborted retry) forever with no tombstone.
 * 90s per PROPOSE_TAKES_MAX_TOKENS of output, floored at 90s, capped at
 * EXTRACTOR_CALL_TIMEOUT_MAX_MS.
 */
function extractorCallTimeoutMs(maxTokens: number): number {
  const scaled = Math.ceil((EXTRACTOR_CALL_TIMEOUT_MS * maxTokens) / PROPOSE_TAKES_MAX_TOKENS);
  return Math.min(EXTRACTOR_CALL_TIMEOUT_MAX_MS, Math.max(EXTRACTOR_CALL_TIMEOUT_MS, scaled));
}

/**
 * #5958: the configured per-call bound, held to the phase time left so it
 * never outlasts the phase deadline (never below the default 90s floor, which
 * the unconfigured path already allows). Unset: no override.
 */
function phaseBoundedCallTimeout(configuredMs: number | undefined, phaseRemainingMs: number): { callBoundMs?: number } {
  if (configuredMs === undefined) return {};
  return { callBoundMs: Math.min(configuredMs, Math.max(EXTRACTOR_CALL_TIMEOUT_MS, phaseRemainingMs)) };
}

/**
 * #3763 — halt streak for a dead extractor lane. When EVERY extractor call in
 * the run has failed (zero successes) and the failure count reaches this
 * streak, the page loop halts instead of burning an LLM call (and its input
 * tokens) on every remaining page. Any single success disarms the halt for
 * the rest of the run — a mixed run is per-page noise, not a dead lane.
 * Deliberately NO failure tombstone (#3910 policy): failed pages retry next
 * cycle once the underlying cause clears.
 */
export const EXTRACTOR_FAILURE_HALT_STREAK = 5;

/**
 * Production extractor — calls gateway.chat with the EXTRACT_TAKES_PROMPT
 * and parses the JSON array output. Returns [] on parse failure (logged as
 * warning, not thrown — one bad page must not abort the phase).
 *
 * Stub-prompt note: the v0.36.1.0 ship-state prompt is a placeholder. Real
 * extractor lands when T19 corpus build produces the tuned prompt. Until
 * then, the production extractor returns whatever the stub LLM produces —
 * empirically often a sparse list or [].
 */
/** Prompt flags read once per phase run: #5425 attribution rules and extraction.date_grounding. */
async function takesPromptFlags(engine: BrainEngine): Promise<{ attributionRules: boolean; dateGrounding: boolean }> {
  const attributionRules = String(await Promise.resolve(engine.getConfig?.('dream.propose_takes.attribution_rules')).catch(() => null) ?? '').trim() === 'true';
  return { attributionRules, dateGrounding: await isConsumerDateGroundingOn(engine, 'takes') };
}

export async function defaultExtractor(
  input: Parameters<ProposeTakesExtractor>[0],
): Promise<ProposedTake[]> {
  const prompt = (input.attributionRules
    ? EXTRACT_TAKES_PROMPT.replace('For each gradeable claim,', `${EXTRACT_TAKES_ATTRIBUTION_RULES}For each gradeable claim,`)
    : EXTRACT_TAKES_PROMPT)
    .replace('{EXISTING_TAKES_JSON}', JSON.stringify(input.existingTakes, null, 2))
    .replace('PAGE PROSE:\n', input.dateGrounding
      ? `${observationDateRule()}\n${observationDateLine(input.observationDate ?? resolveObservationDate({ slug: input.pagePath }))}\n\nPAGE PROSE:\n`
      : 'PAGE PROSE:\n')
    .replace('{PAGE_BODY}', input.pageBody);

  // #4494: per-run configurable caps (dream.propose_takes.max_tokens /
  // .retry_max_tokens), threaded by the phase; the #3763 constants stay as
  // defaults. Retry is clamped >= base so a partial override can't shrink
  // the escalation below the first attempt.
  const baseMaxTokens = Math.max(256, Math.floor(input.maxTokens ?? PROPOSE_TAKES_MAX_TOKENS));
  const retryMaxTokens = Math.max(
    baseMaxTokens,
    Math.floor(input.retryMaxTokens ?? PROPOSE_TAKES_RETRY_MAX_TOKENS),
  );

  // Bound each call so one stalled provider socket can't pin the phase for the
  // full gateway default (GBRAIN_AI_CHAT_TIMEOUT_MS, 300s) x pageLimit. The
  // caller already catches per-page errors, logs a warning, and continues.
  // The bound scales with maxTokens so an escalated or configured larger cap
  // gets time to generate what it allows. An operator bound (callBoundMs)
  // takes its place for every call: claude-cli never reports a truncation,
  // so on that route the retry and its longer scaled bound never happen.
  const call = (maxTokens: number) => {
    const boundMs = input.callBoundMs ?? extractorCallTimeoutMs(maxTokens);
    const ownBound = AbortSignal.timeout(boundMs);
    return gatewayChat({
      messages: [{ role: 'user', content: prompt }],
      ...(input.modelHint ? { model: input.modelHint } : {}),
      maxTokens,
      abortSignal: ownBound,
    }).catch((err: unknown) => {
      // Reword only what our own bound stopped; any other failure, including
      // the gateway's shorter chat timeout, keeps its error untouched. The
      // provider's abort text alone names neither the bound nor the key.
      if (!ownBound.aborted) throw err;
      const providerText = err instanceof Error ? err.message : String(err);
      throw new Error(
        `propose_takes extractor: ${input.pagePath} timed out after ${boundMs} ms, the per-call bound ` +
        `(${providerText}). To move it, set ${PROPOSE_TAKES_CALL_TIMEOUT_KEY} to whole ms from ` +
        `${PROPOSE_TAKES_CALL_TIMEOUT_MIN_MS} to ${PROPOSE_TAKES_CALL_TIMEOUT_MAX_MS}. ` +
        `No tombstone was written; the page is retried next cycle.`,
        { cause: err },
      );
    });
  };
  let result = await call(baseMaxTokens);

  // #3763: a truncated response (stopReason 'length' — e.g. reasoning tokens
  // eating the cap, or a dense page extracting many claims) produced
  // unparseable JSON that the ambiguity guard below rethrew as a GENERIC
  // 'transient — retry', so the page was re-billed at the same too-small cap
  // every cycle forever. Retry ONCE at the escalated cap (#2113 parity);
  // still-truncated throws a message that NAMES the truncation so the phase
  // warning tells the operator what actually happened.
  if (result.stopReason === 'length') {
    process.stderr.write(
      `[propose_takes] WARN: extractor output truncated at maxTokens=${baseMaxTokens} ` +
      `(${input.pagePath}); retrying once at ${retryMaxTokens}\n`,
    );
    result = await call(retryMaxTokens);
    if (result.stopReason === 'length') {
      throw new Error(
        `propose_takes extractor: output truncated (stopReason=length) even at ` +
        `maxTokens=${retryMaxTokens} on ${input.pagePath} — ` +
        `page prose extracts more than the cap can carry; raise ` +
        `dream.propose_takes.max_tokens (thinking models spend reasoning tokens ` +
        `inside this budget); no tombstone written, page retries next cycle`,
      );
    }
  }

  // ChatResult.text is already the concatenated text content. The caller
  // memoizes an empty extraction with a tombstone, so only a reply whose first
  // JSON value is a complete empty array (and that carries no takes after it)
  // counts as "no claims"; anything unparseable is transient and throws, so the
  // phase retries the page next cycle (writing no tombstone).
  const outcome = extractorOutcome(result.text);
  if (outcome.kind === 'invalid') {
    throw new Error('propose_takes extractor: no parseable takes JSON (transient — retry)');
  }
  const takes = outcome.kind === 'valid_nonempty' ? outcome.takes : [];
  // #4737: model_id provenance comes from the RESPONSE, not the request —
  // ChatResult.model is the 'provider:modelId' that actually answered.
  const servedModel =
    typeof result.model === 'string' && result.model.trim() !== '' ? result.model : undefined;
  return servedModel ? takes.map((t) => ({ ...t, served_model: servedModel })) : takes;
}

/**
 * #4736: kinds the pre-fix EXTRACT_TAKES_PROMPT asked for. Cached and
 * old-model outputs still emit them; map them onto the fence vocabulary
 * deterministically so their provenance classifies instead of relying on
 * the blind coerce-to-'take' default.
 */
const LEGACY_EXTRACTOR_KIND_MAP: Record<string, ProposedTake['kind']> = {
  prediction: 'take',
  judgment: 'take',
};

/**
 * #6782: what one extractor reply holds. `valid_empty` only when the reply's
 * first JSON value is a complete empty array and no JSON object, and no array
 * holding a take-shaped object, follows it (a later `[1]` citation does not
 * count); `valid_nonempty` when the first value (an array, or a lone object)
 * yields at least one take; `invalid` for everything else (prose only,
 * truncated or malformed JSON, a claim-less value, or an empty array followed
 * by takes), which the caller retries and never tombstones.
 */
export type ExtractorOutcome = { kind: 'valid_empty' } | { kind: 'valid_nonempty'; takes: ProposedTake[] } | { kind: 'invalid' };

export function extractorOutcome(raw: string): ExtractorOutcome {
  const located = locateFirstJsonValue(raw);
  if (!located) return { kind: 'invalid' };
  if (Array.isArray(located.value) && located.value.length === 0) {
    return carriesLaterTakes(located.rest) ? { kind: 'invalid' } : { kind: 'valid_empty' };
  }
  const takes = takesOf(located.value);
  return takes.length ? { kind: 'valid_nonempty', takes } : { kind: 'invalid' };
}

/**
 * Parse extractor output into ProposedTake[] (the takes of a `valid_nonempty`
 * outcome, else none). Handles common LLM output sins (markdown fence
 * wrapping, leading/trailing prose, single-object instead of array). Kind
 * tokens are case/whitespace-normalized, matched against the fence vocabulary
 * (with the #4736 legacy mapping), and anything else coerces to 'take'.
 */
export function parseExtractorOutput(raw: string): ProposedTake[] {
  const outcome = extractorOutcome(raw);
  return outcome.kind === 'valid_nonempty' ? outcome.takes : [];
}

/**
 * The reply's first JSON array or object after stripping `<think>` reasoning
 * (#2559) and a whole-reply code fence: the whole remainder when it parses,
 * else only up to the value's own closing bracket (#5209), with the text after it.
 */
function locateFirstJsonValue(raw: string): { value: unknown; rest: string } | null {
  if (!raw || raw.trim().length === 0) return null;
  let text = raw.trim().replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
  if (fenced) text = (fenced[1] ?? '').trim();
  const firstArr = text.indexOf('[');
  const firstObj = text.indexOf('{');
  if (firstArr === -1 && firstObj === -1) return null;
  const start = firstArr !== -1 && (firstObj === -1 || firstArr < firstObj) ? firstArr : firstObj;
  try {
    return { value: JSON.parse(text.slice(start)), rest: '' };
  } catch {
    const end = matchingCloseBracket(text, start);
    if (end === -1) return null;
    try {
      return { value: JSON.parse(text.slice(start, end + 1)), rest: text.slice(end + 1) };
    } catch {
      return null;
    }
  }
}

/** True when `text` holds a parseable JSON object, or an array holding a take-shaped object. */
function carriesLaterTakes(text: string): boolean {
  for (let i = 0; i < text.length; i++) {
    if (text[i] !== '[' && text[i] !== '{') continue;
    const end = matchingCloseBracket(text, i);
    if (end === -1) continue;
    let value: unknown;
    try { value = JSON.parse(text.slice(i, end + 1)); } catch { continue; }
    if (!Array.isArray(value) && value !== null && typeof value === 'object') return true;
    if (Array.isArray(value) && value.some(v => v !== null && typeof v === 'object' && typeof (v as Record<string, unknown>).claim_text === 'string')) return true;
    i = end;
  }
  return false;
}

function takesOf(parsed: unknown): ProposedTake[] {
  const arr = Array.isArray(parsed) ? parsed : [parsed];
  const out: ProposedTake[] = [];
  for (const raw of arr) {
    if (typeof raw !== 'object' || raw === null) continue;
    const r = raw as Record<string, unknown>;
    const claim_text = typeof r.claim_text === 'string' ? r.claim_text.trim() : '';
    if (!claim_text || claim_text.length > 500) continue;
    const kindRaw = typeof r.kind === 'string' ? r.kind.trim().toLowerCase() : '';
    const kind = TAKE_KIND_VALUES.has(kindRaw)
      ? (kindRaw as ProposedTake['kind'])
      : (LEGACY_EXTRACTOR_KIND_MAP[kindRaw] ?? 'take');
    const holder = typeof r.holder === 'string' && r.holder.length > 0 ? r.holder : 'brain';
    const weightRaw = typeof r.weight === 'number' ? r.weight : 0.5;
    const weight = Math.max(0, Math.min(1, weightRaw));
    const domain = typeof r.domain === 'string' && r.domain.length > 0 ? r.domain : undefined;
    out.push({ claim_text, kind, holder, weight, domain });
  }
  return out;
}

/**
 * BaseCyclePhase subclass. Walks pages, checks idempotency cache, calls
 * extractor, writes proposals.
 */
/**
 * #4168 — the phase deadline is DERIVED, never a literal. The old
 * PHASE_DEADLINE_MS thirty-minute literal was bit-identical to the
 * autopilot-cycle handler anchor (and the clocks were not even co-started:
 * the job clock starts at claim, this phase starts LATE in ALL_PHASES), so
 * the clean-exit `deadline_hit` path was structurally unreachable in
 * production — cycles died on wall-clock instead of completing partial and
 * `cycle_freshness` never advanced. Same duplicated-literal class as #2781.
 *
 * Fail-loud derivation (autopilot-timeout.ts precedent): a missing handler
 * anchor throws HERE, at module load — which propagates through cycle.ts's
 * dynamic import and fails the WHOLE cycle visibly rather than one phase
 * silently. Accepted trade; the drift-guard test pins the inequality.
 */
function requireCycleAnchorMs(): number {
  const ms = defaultTimeoutMsFor('autopilot-cycle');
  if (ms === null) {
    throw new Error(
      "propose_takes: 'autopilot-cycle' has no entry in HANDLER_DEFAULT_TIMEOUT_MS " +
      '(handler-timeouts.ts) — the phase deadline can no longer be derived from it. See #4168.',
    );
  }
  return ms;
}

/** Headroom for grade_takes + calibration_profile, which run AFTER this
 *  phase in the same calibration block with no deadline of their own. */
export const PHASE_DEADLINE_FRACTION_OF_JOB = 0.8;
export const PROPOSE_TAKES_FALLBACK_DEADLINE_MS = Math.floor(
  requireCycleAnchorMs() * PHASE_DEADLINE_FRACTION_OF_JOB,
);
/** Mirrors MIN_PATTERNS_SUBAGENT_BUDGET_MS: below this the phase cannot do
 *  useful LLM work before the job's kill switch — skip honestly instead. */
export const MIN_PROPOSE_TAKES_BUDGET_MS = 2 * 60 * 1000;

/**
 * Resolve the phase's wall-clock budget from the REAL remaining job time
 * when it is known. Shaped like patterns.ts's clampSubagentBudgets: null
 * means "not worth starting" (caller returns an honest skip). Pure —
 * unit-testable without an engine.
 */
export function resolveProposeTakesDeadlineMs(
  deadlineAtMs: number | null | undefined,
  nowMs: number,
): number | null {
  if (deadlineAtMs == null) return PROPOSE_TAKES_FALLBACK_DEADLINE_MS;
  const remaining = deadlineAtMs - CYCLE_DEADLINE_RESERVE_MS - nowMs;
  // Red-team + adversarial F4: the grade_takes/calibration_profile headroom
  // the 0.8 fraction exists for must apply on the THREADED path too, and the
  // MIN floor must gate the FRACTIONED value — clamping a sub-MIN fraction
  // back UP to MIN would hand propose_takes the whole remaining window and
  // start the downstream phases inside the reserve. Under the floor, skip
  // honestly instead.
  const fractioned = Math.floor(remaining * PHASE_DEADLINE_FRACTION_OF_JOB);
  if (fractioned < MIN_PROPOSE_TAKES_BUDGET_MS) return null;
  return Math.min(fractioned, PROPOSE_TAKES_FALLBACK_DEADLINE_MS);
}

class ProposeTakesPhase extends BaseCyclePhase {
  readonly name = 'propose_takes' as CyclePhase;
  protected readonly budgetUsdKey = 'cycle.propose_takes.budget_usd';
  protected readonly budgetUsdDefault = 5.0;

  protected override mapErrorCode(err: unknown): string {
    if (err instanceof GBrainError) return err.problem;
    if (err instanceof Error) {
      if (err.message.includes('content_hash')) return 'CALIBRATION_PROPOSAL_DEDUP_FAIL';
      if (err.message.includes('budget') || err.message.includes('Budget')) return 'CALIBRATION_GRADE_BUDGET_EXHAUSTED';
    }
    return 'PROPOSE_TAKES_UNKNOWN';
  }

  protected async process(
    engine: BrainEngine,
    scope: ScopedReadOpts,
    _ctx: OperationContext,
    opts: ProposeTakesOpts,
  ): Promise<{ summary: string; details: Record<string, unknown>; status?: PhaseStatus }> {
    // #4102 — off switch. The phase is ON by default (it ships in the default
    // phase list), but `gbrain config set cycle.propose_takes.enabled false`
    // must actually stop the LLM spend. Only an EXPLICIT falsy value skips
    // (unset = default on, fail-open on read errors so a config-plane blip
    // never silently disables the phase); `--once` bypasses for one run.
    if (!opts.once) {
      let enabledRaw: string | null = null;
      try {
        enabledRaw = await engine.getConfig?.('cycle.propose_takes.enabled') ?? null;
      } catch {
        enabledRaw = null;
      }
      if (enabledRaw != null && !isConfigTruthy(enabledRaw)) {
        return {
          summary: 'propose_takes skipped: cycle.propose_takes.enabled=false',
          details: {
            reason: 'disabled',
            enable_hint: 'gbrain config set cycle.propose_takes.enabled true',
            pages_scanned: 0,
            cache_hits: 0,
            cache_misses: 0,
            proposals_inserted: 0,
            tombstones_written: 0,
            budget_exhausted: false,
            warnings: [],
          },
          status: 'skipped',
        };
      }
    }

    const extractor = opts.extractor ?? defaultExtractor;
    const [{ attributionRules, dateGrounding }, attributionChecks] = await Promise.all([takesPromptFlags(engine), attributionChecksEnabled(engine)]);
    const promptVersion = opts.promptVersion ?? `${PROPOSE_TAKES_PROMPT_VERSION}${attributionRules ? PROPOSE_TAKES_ATTRIBUTION_PROMPT_SUFFIX : ''}`;
    const pageLimit = opts.pageLimit ?? 100;
    const skipPagesWithFence = opts.skipPagesWithFence ?? false;
    // gbrain#4168: explicit test override wins; otherwise the REAL remaining
    // job budget (when the cycle threads deadlineAtMs) clamped to the derived
    // fallback. At the default installed-daemon interval the old 30-min
    // literal was bit-identical to the job timeout floor, and since this
    // phase starts after earlier phases, phase-elapsed always trailed
    // job-elapsed — the clean partial-exit below was unreachable and cycles
    // dead-lettered instead of banking work. Resolved to null = not enough
    // budget to start (see the honest-skip return after the provider probe).
    const resolvedDeadlineMs =
      opts.deadlineMs ?? resolveProposeTakesDeadlineMs(opts.deadlineAtMs, Date.now());
    const phaseStartMs = Date.now();
    const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`;

    const modelId = opts.model ?? getChatModel();

    // #4494: configurable extractor output caps (dream.triage.max_tokens
    // precedent — floor 256, retry clamped >= base, fail-open to the #3763
    // defaults on any config-plane error). Thinking models spend reasoning
    // tokens inside maxTokens, so the hardcoded 2048/4096 pair put dense
    // pages into a permanent truncate → retry → truncate → re-bill loop.
    let extractorMaxTokens = PROPOSE_TAKES_MAX_TOKENS;
    let extractorRetryMaxTokens = PROPOSE_TAKES_RETRY_MAX_TOKENS;
    try {
      const readCap = async (key: string): Promise<number | null> => {
        const raw = await engine.getConfig?.(key);
        if (raw == null || String(raw).trim() === '') return null;
        const n = Number(raw);
        return Number.isFinite(n) ? n : null;
      };
      const baseCap = await readCap('dream.propose_takes.max_tokens');
      if (baseCap != null) extractorMaxTokens = Math.max(256, Math.floor(baseCap));
      const retryCap = await readCap('dream.propose_takes.retry_max_tokens');
      if (retryCap != null) extractorRetryMaxTokens = Math.floor(retryCap);
    } catch { /* keep defaults */ }
    extractorRetryMaxTokens = Math.max(extractorMaxTokens, extractorRetryMaxTokens);
    const callTimeout = await readPhaseConfigNumber(engine, PROPOSE_TAKES_CALL_TIMEOUT_KEY); // #5874

    // With the default (gateway) extractor, skip cheaply when the resolved
    // model's provider can't run: unknown provider/model, or any keyed provider
    // without its key (#6705; same probe as patterns.ts). An injected
    // extractor bypasses the gateway, so it is never gated. (Takeover of
    // PR #1979's intent by @shawnduggan.)
    if (!opts.extractor) {
      const probe = probeChatModelWithKey(normalizeModelId(modelId));
      if (!probe.ok) {
        return {
          summary: `propose_takes skipped: ${probe.detail}`,
          details: {
            reason: 'no_provider',
            model: modelId,
            pages_scanned: 0,
            cache_hits: 0,
            cache_misses: 0,
            proposals_inserted: 0,
            budget_exhausted: false,
            warnings: [],
          },
          status: 'skipped',
        };
      }
    }

    // #4168 honest skip — placed AFTER the cheap provider probe (patterns.ts
    // ordering precedent) and BEFORE any rollup/DB write, matching the
    // no_provider skip: an insufficient-budget run records neither a halt
    // nor a completed round. On a brain where earlier phases eat the whole
    // job budget this fires EVERY cycle — the reason string and operator
    // hint are load-bearing observability, not decoration (a repeated-skip
    // doctor check is a filed follow-up).
    if (resolvedDeadlineMs === null) {
      return {
        summary:
          `propose_takes skipped: remaining cycle budget under ` +
          `${Math.round(MIN_PROPOSE_TAKES_BUDGET_MS / 1000)}s ` +
          `(reserve ${Math.round(CYCLE_DEADLINE_RESERVE_MS / 1000)}s) — earlier phases consumed ` +
          `the job budget; raise the autopilot interval or the autopilot-cycle handler anchor ` +
          `if this repeats every cycle. Next cycle retries with a fresh budget.`,
        details: {
          reason: 'insufficient_cycle_budget',
          // The job deadline is WHY the phase can't start — carry the same
          // flag the mid-run partial exit sets so dashboards see one signal.
          deadline_hit: true,
          pages_scanned: 0,
          cache_hits: 0,
          cache_misses: 0,
          proposals_inserted: 0,
          tombstones_written: 0,
          budget_exhausted: false,
          warnings: [],
        },
        status: 'skipped',
      };
    }
    const deadlineMs = resolvedDeadlineMs;

    const result: ProposeTakesResult = {
      pages_scanned: 0,
      cache_hits: 0,
      cache_misses: 0,
      proposals_inserted: 0,
      tombstones_written: 0,
      rejected_claims_skipped: 0,
      budget_exhausted: false,
      llm_calls_succeeded: 0,
      llm_calls_failed: 0,
      failed_pages: [],
      failure_tombstones_written: 0,
      warnings: callTimeout.warning ? [callTimeout.warning] : [],
      deadline_hit: false,
    };

    // gbrain#4168: job budget already inside the reserve window — exit
    // cleanly before ANY work (the in-loop `elapsed > deadline` check can't
    // fire on the first iteration when the effective deadline is 0).
    if (deadlineMs <= 0) {
      result.warnings.push('phase skipped: job deadline already inside the reserve window');
      result.deadline_hit = true;
      return {
        summary: `propose_takes: skipped — job deadline inside the reserve window (run ${proposalRunId})`,
        details: { ...result, proposal_run_id: proposalRunId, prompt_version: promptVersion },
        status: 'warn' as PhaseStatus,
      };
    }

    // Load pages eligible for proposal. Source-scoped per BaseCyclePhase; #5213 types narrow it.
    // GBRA-130 A-25: a page that failed before at this content hash and prompt
    // version is retried after the fresh pages, so failing pages at the head of
    // the list cannot halt the run before the rest are tried.
    const listed = await listCandidatePages(engine, scope, pageLimit, await proposeTakesTypes(engine));
    const failures = await ProposeTakesFailures.load(engine, listed.map(page => page.source_id ?? scope.sourceId ?? 'default'), promptVersion);
    const pages = failures.order(listed, page => [page.source_id ?? scope.sourceId ?? 'default', contentHash(page.compiled_truth ?? '')]);

    if (opts.reporter) {
      opts.reporter.start('propose_takes.pages' as never, pages.length);
    }

    // #3044 — shared halt policy: auth/billing halt on the first hit, a
    // rate_limit streak halts after RATE_LIMIT_HALT_STREAK consecutive
    // failures. A successful call resets the streak.
    const llmHalt = createGlobalLlmHaltTracker();

    for (const page of pages) {
      // Phase deadline check. Break (not throw) so the phase returns a
      // partial result with deadline_hit:true; work already banked stays.
      const elapsedMs = Date.now() - phaseStartMs;
      if (elapsedMs > deadlineMs) {
        result.warnings.push(
          `phase deadline hit at page ${result.pages_scanned}/${pages.length} ` +
          `after ${(elapsedMs / 1000).toFixed(0)}s (cap ${(deadlineMs / 1000).toFixed(0)}s); partial completion`,
        );
        result.deadline_hit = true;
        break;
      }

      result.pages_scanned += 1;
      this.tick(opts);

      // Skip pages that have NO prose body (e.g. metadata-only entity stubs).
      const body = page.compiled_truth ?? '';
      if (body.trim().length === 0) continue;
      if (skipPagesWithFence && hasCompleteFence(body)) continue;

      const ch = contentHash(body);
      const existingTakes = extractExistingTakesForDedup(body);

      // Idempotency check. If a row exists for (source_id, page_slug, content_hash,
      // prompt_version), this page was already processed — skip and count as cache hit.
      const sourceId = page.source_id ?? scope.sourceId ?? 'default';
      const cached = await engine.executeRaw<{ id: number }>(
        `SELECT id FROM take_proposals
         WHERE source_id = $1 AND page_slug = $2 AND content_hash = $3 AND prompt_version = $4
         LIMIT 1`,
        [sourceId, page.slug, ch, promptVersion],
      );
      if (cached.length > 0) {
        result.cache_hits += 1;
        continue;
      }
      result.cache_misses += 1;

      // Budget pre-check before the LLM call. Estimate: ~1500 input tokens + 500 output.
      const budget = this.checkBudget({
        modelId,
        estimatedInputTokens: 1500,
        maxOutputTokens: 500,
      });
      if (!budget.allowed) {
        result.budget_exhausted = true;
        result.warnings.push(
          `budget exhausted at page ${result.pages_scanned}/${pages.length} (cumulative $${budget.cumulativeCostUsd.toFixed(4)} / cap $${budget.budgetUsd.toFixed(2)})`,
        );
        break;
      }

      // Call the extractor. Per-page errors log a warning and continue —
      // UNLESS they classify as a whole-run condition (#3044): auth/billing
      // halts on the first hit (a revoked key or exhausted spend limit fails
      // identically on every remaining page); a bare rate_limit halts only
      // after RATE_LIMIT_HALT_STREAK consecutive hits (a burst 429 can clear
      // between pages).
      let proposals: ProposedTake[];
      try {
        proposals = await extractor({
          pagePath: page.slug,
          pageBody: publicBody(body),
          existingTakes,
          modelHint: opts.model,
          // #4494: configurable output caps (see resolution above).
          maxTokens: extractorMaxTokens,
          retryMaxTokens: extractorRetryMaxTokens,
          ...phaseBoundedCallTimeout(callTimeout.value, deadlineMs - (Date.now() - phaseStartMs)),
          attributionRules, dateGrounding,
        });
      } catch (err) {
        // GBRA-130: a run-scoped error (abort, lost connection, failed
        // transaction) stops the phase and records nothing. Every failure feeds
        // the breaker except a recorded page repeating its content-class code.
        failures.fail(sourceId, page.slug, ch, err);
        result.llm_calls_failed += 1;
        const msg = err instanceof Error ? err.message : String(err);
        const detail = `extractor failed on ${page.slug}: ${msg}`;
        const decision = llmHalt.observe(err);
        if (decision !== 'continue') {
          result.aborted_global_error = haltedClassOf(decision)!;
          result.warnings.push(
            `aborting phase at page ${result.pages_scanned}/${pages.length}: ` +
            `${llmHalt.note()} (${detail})`,
          );
          break;
        }
        result.warnings.push(detail);
        // #3763: N consecutive failures with ZERO successes = dead lane.
        // Halt instead of spending an LLM call on every remaining page. A
        // single success anywhere in the run keeps llm_calls_succeeded > 0
        // and permanently disarms this halt (mixed runs are per-page noise).
        // GBRA-130 A-25: a recorded page repeating its content-class code is the
        // expected outcome and does not count; a transient or provider failure
        // always does, so an outage halts even on pages that failed before.
        if (
          result.llm_calls_succeeded === 0 &&
          failures.freshFailures >= EXTRACTOR_FAILURE_HALT_STREAK
        ) {
          result.aborted_failure_streak = true;
          result.warnings.push(
            `aborting phase at page ${result.pages_scanned}/${pages.length}: ` +
            `${result.llm_calls_failed} consecutive extractor failures with zero successes — ` +
            `halting to avoid re-billing every remaining page (no tombstones written; pages retry next cycle)`,
          );
          break;
        }
        continue;
      }
      result.llm_calls_succeeded += 1;
      failures.succeed(sourceId, page.slug, ch);
      llmHalt.reset();
      if (attributionChecks) proposals = downgradeAssistantOnlyHolders(proposals, body);
      const fold = (text: string) => text.replace(/\s+/g, ' ').trim().toLowerCase();
      const heldTakes = existingTakes.filter(take => take.holder !== 'world').map(take => fold(take.claim)).filter(claim => claim.length >= 12);
      proposals = proposals.filter(p => !heldTakes.some(claim => fold(p.claim_text).includes(claim)));
      // #7299: a new content hash extracts the page again; a claim the operator already rejected on it stays rejected.
      const rejected = new Set((await engine.executeRaw<{ claim_text: string }>(
        `SELECT claim_text FROM take_proposals
          WHERE source_id = $1 AND page_slug = $2 AND status = 'rejected' AND acted_at IS NOT NULL`,
        [sourceId, page.slug],
      )).map(row => row.claim_text));
      const fresh = proposals.filter(p => !rejected.has(p.claim_text));
      result.rejected_claims_skipped += proposals.length - fresh.length;
      proposals = fresh;

      // Write proposals to take_proposals. #2138: the idempotency key is
      // per-CLAIM — take_proposals_idempotency_idx folds md5(claim_text) into
      // the per-page tuple (migration v125), so a multi-claim page keeps every
      // claim. RETURNING id prevents a repeated claim from inflating the count.
      for (const p of proposals) {
        const inserted = await engine.executeRaw<{ id: number }>(
          `INSERT INTO take_proposals
             (source_id, page_slug, content_hash, prompt_version, proposal_run_id,
              claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id)
           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::text::jsonb, $12)
           ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING
           RETURNING id`,
          [
            sourceId,
            page.slug,
            ch,
            promptVersion,
            proposalRunId,
            p.claim_text,
            p.kind,
            p.holder,
            p.weight,
            p.domain ?? null,
            JSON.stringify(existingTakes),
            // #4737: prefer the response-derived model (what actually
            // answered) over the requested one for provenance.
            p.served_model ?? modelId,
          ],
        );
        result.proposals_inserted += inserted.length;
      }

      // Memoize the empty case too. A page that extracted zero claims gets
      // NO row from the loop above, so without this its idempotency tuple is
      // never recorded and the next cycle re-spends an LLM call on unchanged
      // prose (the idle-cost bug). Write one tombstone row keyed by the same
      // per-page tuple (the cache-hit lookup above matches ANY row for the
      // 4-tuple; the unique index — take_proposals_idempotency_idx, migration
      // v125 — folds md5(claim_text) in, so the conflict target must too).
      // status='rejected' keeps it out of any pending-review query; its sole
      // purpose is to make the next cycle a cache hit. Only reached on a
      // SUCCESSFUL empty extract — the extractor-throw path `continue`s above,
      // so failed pages are retried rather than tombstoned.
      if (proposals.length === 0) {
        await engine.executeRaw(
          `INSERT INTO take_proposals
             (source_id, page_slug, content_hash, prompt_version, proposal_run_id,
              claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id, status)
           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::text::jsonb, $12, 'rejected')
           ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING`,
          [
            sourceId,
            page.slug,
            ch,
            promptVersion,
            proposalRunId,
            EMPTY_EXTRACTION_TOMBSTONE_TEXT,
            'fact',
            'brain',
            0,
            null,
            JSON.stringify(existingTakes),
            modelId,
          ],
        );
        result.tombstones_written += 1;
      }
    }

    if (opts.reporter) opts.reporter.finish();

    // GBRA-130 A-25: fold this run into the failure ledgers. A page failing the
    // same content-class way in PROPOSE_TAKES_HOLD_RUNS counted runs gets a
    // failure tombstone (the idempotency check then skips it until its content
    // or the prompt version changes); a systemic run or a whole-run LLM halt
    // counts nothing.
    const settled = await failures.settle(engine, { counted: result.aborted_global_error === undefined, proposalRunId, modelId });
    result.failed_pages = settled.failed;
    result.failure_tombstones_written = settled.tombstones;
    if (settled.systemic) result.systemic = settled.systemic;

    const halted = await recordProposeTakesRun(engine, result, scope.sourceId ?? 'default', proposalRunId);

    // Status folds warnings in (the extract_facts precedent from #1928): a
    // run with swallowed per-page failures must not read as a clean 'ok'.
    // Severity split (#3044): a global halt with ZERO successful extractor
    // calls means the whole LLM lane is down — that is a phase 'fail', not a
    // 'warn' (deriveStatus turns one failed phase into a 'partial' cycle;
    // the autopilot handler deliberately does not throw on partial). A halt
    // after some successes is a partial run → 'warn'.
    const warningCount = result.warnings.length;
    // #3763: an all-failures streak halt is the same severity as a
    // zero-success global halt — the whole extractor lane is down.
    const phaseFailed =
      (result.aborted_global_error !== undefined && result.llm_calls_succeeded === 0) ||
      result.aborted_failure_streak === true;
    return {
      summary:
        `propose_takes: scanned ${result.pages_scanned} pages, ${result.cache_hits} cached, ${result.proposals_inserted} new proposals, ${result.tombstones_written} empty (run ${proposalRunId})` +
        (result.rejected_claims_skipped > 0
          ? `; ${result.rejected_claims_skipped} claim(s) rejected before on the same page not proposed again`
          : '') +
        (result.failure_tombstones_written > 0
          ? `; ${result.failure_tombstones_written} page(s) held after ${PROPOSE_TAKES_HOLD_RUNS} failed runs (gbrain takes propose --failed)`
          : '') +
        (result.aborted_global_error
          ? `; aborted on ${result.aborted_global_error} error after ${result.pages_scanned} page(s)`
          : '') +
        (result.aborted_failure_streak
          ? `; aborted after ${result.llm_calls_failed} consecutive extractor failures (zero successes)`
          : '') +
        (warningCount > 0 ? ` (${warningCount} warning(s))` : ''),
      details: { ...result, halted, proposal_run_id: proposalRunId, prompt_version: promptVersion },
      status: phaseFailed ? 'fail' : halted || warningCount > 0 ? 'warn' : 'ok',
    };
  }
}

/** The takes.proposed receipt and rollup of one run; returns whether the run halted (any cap or halt). */
async function recordProposeTakesRun(engine: BrainEngine, result: ProposeTakesResult, sourceIdForReceipt: string, proposalRunId: string): Promise<boolean> {
  // v0.42 Wave B3: receipt + rollup for propose_takes. Source-scoped
  // via the read scope. Receipt only when proposals actually written.
  // Managed brains skip the receipt page (a legacy putPage the coordinator
  // refuses), like extract_atoms and synthesize_concepts; the rollup stays.
  if (result.proposals_inserted > 0 && !await managedPersistenceEnabled(engine)) {
    try {
      await writeReceipt(engine, {
        kind: 'takes.proposed',
        source_id: sourceIdForReceipt,
        run_id: proposalRunId,
        round: 'single',
        extracted_at: new Date().toISOString(),
        total_rows: result.proposals_inserted,
        cost_usd: 0, // tracker isn't exposed at this layer; cost tracked centrally
        summary:
          `Proposed ${result.proposals_inserted} new takes from ${result.pages_scanned} pages ` +
          `(${result.cache_hits} cached).`,
      });
    } catch (err) {
      console.error(`[propose_takes] receipt write failed: ${(err as Error).message}`);
    }
  }
  // #4482: three-way stop classification. A budget/deadline cap is the
  // extractor working as designed (partial progress banked; the rest
  // drains over future runs) — recorded as expected_limit_delta, a
  // capacity signal doctor's failure rate excludes. A global-error abort
  // (#3044) or an all-failures streak (#3763) is a REAL halt, unchanged
  // from today. An error alongside a cap counts as the error.
  // `halted` (any incomplete round, caps included) is kept for the phase
  // result's status/details below — the diagnostic split is rollup-only.
  const halted =
    result.budget_exhausted ||
    result.deadline_hit === true ||
    result.aborted_global_error !== undefined ||
    result.aborted_failure_streak === true;
  await upsertExtractRollup(engine, {
    kind: 'takes.proposed',
    source_id: sourceIdForReceipt,
    ...classifyRunStop({
      budget_exhausted: result.budget_exhausted === true,
      deadline_hit: result.deadline_hit === true,
      error:
        result.aborted_global_error !== undefined ? providerHaltReason(result.aborted_global_error) ?? 'item_error'
          : result.aborted_failure_streak === true ? 'item_error' : false,
    }),
  });
  return halted;
}

/**
 * Public entry point — mirrors the v0.23 `runPhaseSynthesize` shape so the
 * cycle orchestrator in cycle.ts can call it uniformly.
 */
export async function runPhaseProposeTakes(
  ctx: OperationContext,
  opts: ProposeTakesOpts = {},
) {
  return new ProposeTakesPhase().run(ctx, opts);
}

/** Test-only access to the class for subclassing in tests. */
export const __testing = {
  ProposeTakesPhase,
  parseExtractorOutput,
  contentHash,
  hasCompleteFence,
  extractExistingTakesForDedup,
  listCandidatePages,
};