Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
35.4 KB
Raw Blame
/**
 * v0.31 — Dream-cycle `consolidate` phase: facts → takes promotion.
 *
 * Per /plan-eng-review Phase 5:
 *
 *   For each (source_id, entity_slug) bucket of unconsolidated active facts
 *   the user asserted (or with no recorded speaker):
 *     1. Skip if count < 3 OR oldest fact age < 24h.
 *     2. Cluster by embedding cosine — greedy threshold 0.85.
 *     3. For each cluster ≥ 2: pick the highest-confidence fact's text as
 *        the take claim (v0.31 ships without LLM synthesis to keep the
 *        cycle deterministic; see TODO at the bottom for the v0.32 Sonnet
 *        rewrite).
 *     4. Resolve entity_slug → pages.slug. If the page is missing, skip
 *        this cluster (no auto-page-creation in v0.31; the take needs a
 *        home).
 *     5. INSERT into takes(kind='fact', holder='self', source=concatenated
 *        source_sessions). row_num = MAX existing for the page + 1.
 *     6. UPDATE contributing facts: consolidated_at = now() +
 *        consolidated_into = takes.id. NEVER DELETE.
 *
 * The phase's totals contribute to the runCycle CycleReport via
 * extractTotals (cycle.ts) — facts_consolidated + takes_written.
 */

import type { Action } from '../../agent-output.ts';
import type { BrainEngine, FactRow } from '../../engine.ts';
import type { PhaseResult } from '../../cycle.ts';
import { cosineSimilarity } from '../../facts/classify.ts';
import { claimsDiverge, EXPLICIT_DUPLICATE_THRESHOLD } from '../../facts/capture-dedup.ts';
import { sameInstantTies, type SameInstantTie } from '../../facts/claim-ties.ts';
import { createHash } from 'node:crypto';
import { isAborted } from '../../abort-check.ts';
import { classifyItemFailure, ItemRun } from '../../item-failure.ts';
import { consolidationCloses, maintenancePreflight, submitMaintenanceConsolidation, type MaintenanceAuthority } from '../../persistence/prepared-maintenance.ts';
import { clearConsolidationTies, consolidationRestored, factPreimages, holdConsolidationTie, recordConsolidationGeneration } from '../../generations/adapters/fact-rows.ts';
import { managedPersistenceEnabled } from '../../persistence/ownership.ts';
import { DreamWriteDeferralPass } from '../dream-write-deferral.ts';
import { declareDerivation, derivedMaintenanceTransaction, derivedWriteTrust, isExternalTier, readTaintInputs } from '../../trust/taint.ts';
import { DEFAULT_WRITE_GATE_CONFIG, type WriteGateConfig } from '../../write-gate.ts';
import { loadImportSanityConfig } from '../../import-screen.ts';
import { decideTakeWrite, recordFlaggedRow, recordWriteGateHold, type GatedRowDecision } from '../../write-gate-store.ts';
import type { TaintInput, WriteTrust } from '../../trust/tier.ts';
import { readersSourcedFactSql } from '../../access/member-facts.ts';

export interface ConsolidatePhaseOpts {
  dryRun?: boolean;
  /** In-phase keepalive callback. Awaited between buckets. */
  yieldDuringPhase?: () => Promise<void>;
  /**
   * #1972: cooperative-abort signal. Checked at the top of the bucket loop so a
   * long consolidate relinquishes its worker slot well under the 30s
   * force-evict instead of running to completion after cancellation.
   */
  signal?: AbortSignal;
  /**
   * Cosine cluster threshold for typed (metric) facts. Overrides
   * `cycle.consolidate.cluster_threshold`; default 0.85. Untyped facts cluster
   * at `max(threshold, EXPLICIT_DUPLICATE_THRESHOLD)` (#6023).
   */
  clusterThreshold?: number;
  /**
   * Trust of the caller reading a dry-run preview. Fail-closed: only
   * `remote === false` (the local CLI) sees the text of private facts in
   * `details.clusters`; anything else sees their ids and visibility only.
   */
  remote?: boolean;
  /** Minimum facts per (source, entity) bucket before consolidation. Default 3. */
  minFactsPerBucket?: number;
  /** Minimum age (ms) of the OLDEST fact in a bucket before consolidation. Default 24h. */
  minOldestAgeMs?: number;
  sourceId?: string;
  /** `gbrain dream --phase consolidate --once`: run even when `cycle.consolidate.enabled` is false. */
  once?: boolean;
}

/**
 * #6667: `cycle.consolidate.enabled` (default on) is the off switch for this
 * phase, which promotes unreviewed facts into takes. Only an explicit falsy
 * value skips it; unset, a truthy value and a config read failure keep it on.
 */
export const CONSOLIDATE_ENABLED_KEY = 'cycle.consolidate.enabled';

async function disabledSkip(engine: BrainEngine, opts: ConsolidatePhaseOpts): Promise<PhaseResult | null> {
  if (opts.once || !/^\s*(false|0|off|no)\s*$/i.test(await engine.getConfig(CONSOLIDATE_ENABLED_KEY).catch(() => null) ?? '')) return null;
  return { phase: 'consolidate', status: 'skipped', duration_ms: 0, summary: `consolidate skipped: ${CONSOLIDATE_ENABLED_KEY}=false`,
    details: { reason: 'disabled', enable_hint: `gbrain config set ${CONSOLIDATE_ENABLED_KEY} true` } };
}

/** At most this many failed buckets are listed in `details.bucket_failures`; `buckets_failed` counts them all. */
const BUCKET_FAILURE_SAMPLES = 10;
type BucketFailure = { source_id: string; entity_slug: string; code: string; message: string; fix: Action };

const rerunFix = (sourceId: string): Action => ({ argv: ['gbrain', 'dream', '--phase', 'consolidate', '--source', sourceId], consent: [], actor: 'agent',
  requires_exclusive: false, why: 'Reruns consolidate for this source; the bucket consolidates once its entity and evidence pages are readable again.' });

/**
 * W3 (#7587): consolidations an earlier pass left pending that ended failed, as bucket failures (samples pushed onto
 * `failures`); returns how many. A source whose receipt is still pending submits nothing this pass (`hold`), and a
 * write still pending after its wait defers the rest of its source (`defer`); deferral off restores the failing phase.
 */
function reconciledBucketFailures(deferral: DreamWriteDeferralPass | null, failures: BucketFailure[]): number {
  const failed = deferral?.reconcile.failed ?? [];
  for (const entry of failed.slice(0, Math.max(0, BUCKET_FAILURE_SAMPLES - failures.length))) {
    failures.push({ source_id: entry.source_id, entity_slug: entry.subject.slice(entry.source_id.length + 1), code: entry.code,
      message: `The consolidation an earlier pass left pending (request_id ${entry.request_id}) ended with ${entry.code}: ${entry.message}`.slice(0, 300), fix: rerunFix(entry.source_id) });
  }
  return failed.length;
}

/** Samples a failed bucket into `details.bucket_failures`, at most BUCKET_FAILURE_SAMPLES of them. */
function sampleBucketFailure(failures: BucketFailure[], failure: BucketFailure): void {
  if (failures.length < BUCKET_FAILURE_SAMPLES) failures.push(failure);
}

export const CLUSTER_THRESHOLD_KEY = 'cycle.consolidate.cluster_threshold';
export const DEFAULT_CLUSTER_THRESHOLD = 0.85;

/**
 * v0.35.4 (D-CDX-4 part 2) — chronological valid_until writeback.
 * Sort the cluster by (valid_from ASC, id ASC); stamp each fact's
 * valid_until = the valid_from of the next fact that says something
 * else, by claim text and typed value (consolidationCloses). A restatement of the same claim does not
 * end the fact before it; the newest value keeps valid_until = NULL. This makes the facts
 * table a proper bitemporal record without the contradiction probe
 * having to mutate it (preserves auto-supersession.ts:4 invariant —
 * see also R8 test guard).
 *
 * Idempotent: re-running on the same cluster produces the same
 * chronological order and the same valid_until values. No-op if
 * valid_until is already correct.
 */
async function writeValidUntilChain(tx: BrainEngine, cluster: FactRow[]): Promise<void> {
  const chronological = [...cluster].sort((a, b) => {
    const t = a.valid_from.getTime() - b.valid_from.getTime();
    if (t !== 0) return t;
    return a.id - b.id;
  });
  const values = new Map((await tx.executeRaw<{ id: number; claim_value: number | string | null }>(
    'SELECT id, claim_value FROM facts WHERE id = ANY($1::bigint[])', [cluster.map(f => f.id)])).map(r => [Number(r.id), r.claim_value]));
  for (const { member: older, next: newer } of consolidationCloses(chronological, f => ({ text: f.fact, value: values.get(f.id) }))) {
    await tx.executeRaw(
      // Only UPDATE when the new value would actually change. Avoids
      // touching updated_at on no-op rewrites and keeps idempotency
      // observable in the DB (zero affected rows on stable re-run).
      `UPDATE facts
         SET valid_until = $1
       WHERE id = $2
         AND (valid_until IS DISTINCT FROM $1)`,
      [newer.valid_from, older.id],
    );
  }
}

/**
 * #5363: parse `cycle.consolidate.cluster_threshold`. A cosine similarity in
 * (0, 1]; lower values merge more facts into one take. Throws on anything
 * else so `gbrain config set` can refuse it.
 */
export function parseClusterThreshold(value: string): number {
  const n = Number(value.trim());
  if (value.trim() === '' || !Number.isFinite(n) || n <= 0 || n > 1) {
    throw new Error(`${CLUSTER_THRESHOLD_KEY} must be a number greater than 0 and at most 1 (default ${DEFAULT_CLUSTER_THRESHOLD}); got '${value}'.`);
  }
  return n;
}

/** GST-359 C2: `facts` without those on a page that names readers (a member page): they never become a take on the entity page. */
async function withoutReadersSourced(engine: BrainEngine, facts: FactRow[]): Promise<FactRow[]> {
  const rows = await engine.executeRaw<{ id: number }>(`SELECT id FROM facts WHERE id = ANY($1::bigint[]) AND ${readersSourcedFactSql('facts')}`, [facts.map(f => f.id)]);
  const restricted = new Set(rows.map(row => Number(row.id)));
  return facts.filter(f => !restricted.has(f.id));
}

export async function runPhaseConsolidate(
  engine: BrainEngine,
  opts: ConsolidatePhaseOpts = {},
): Promise<PhaseResult> {
  const disabled = await disabledSkip(engine, opts);
  if (disabled) return disabled;
  const dryRun = opts.dryRun === true;
  const managed = await managedPersistenceEnabled(engine);
  const { threshold, thresholdInvalid } = await resolveClusterThreshold(engine, opts.clusterThreshold);
  const minPerBucket = opts.minFactsPerBucket ?? 3;
  const minOldestAgeMs = opts.minOldestAgeMs ?? 24 * 60 * 60 * 1000;

  let factsConsolidated = 0;
  let takesWritten = 0;
  let bucketsProcessed = 0;
  let bucketsSkipped = 0;
  let clustersSkippedRetired = 0;
  let clustersGateHeld = 0;
  let clustersGateRejected = 0;
  let clustersHeldSameInstant = 0;
  const sameInstantClusters: SameInstantCluster[] = [];
  // #6023: a dry run lists the clusters it would write so an operator can
  // inspect them before `--yes`; capped so the envelope stays bounded.
  const preview: ConsolidateClusterPreview[] = [];
  const showPrivate = opts.remote === false;
  // #5575 write gate: unmanaged takes are gated here (managed ones in takes-prepare); read once per run.
  let gateConfig: WriteGateConfig | undefined;

  // Pull every (source_id, entity_slug) bucket of unconsolidated facts whose entity is a live page (a take needs one, and
  // GBRA-574: an unbacked name key has none, so its bucket is never read). Uses the partial idx_facts_unconsolidated index.
  let buckets: Array<{ source_id: string; entity_slug: string; count: number }>;
  try {
    buckets = await engine.executeRaw<{
      source_id: string; entity_slug: string; count: number;
    }>(`
      SELECT source_id, entity_slug, COUNT(*)::int AS count
      FROM facts
      WHERE consolidated_at IS NULL
        AND expired_at IS NULL
        AND (valid_until IS NULL OR valid_until > now())
        AND entity_slug IS NOT NULL
        AND EXISTS (SELECT 1 FROM pages bucket_page WHERE bucket_page.source_id = facts.source_id AND bucket_page.slug = facts.entity_slug AND bucket_page.deleted_at IS NULL)
        AND (attributed_to IS NULL OR attributed_to = 'user')
        AND ($2::boolean=false OR visibility='world')
        AND ($1::text IS NULL OR source_id=$1) AND NOT ${readersSourcedFactSql('facts')}
      GROUP BY source_id, entity_slug
      HAVING COUNT(*) >= $3
    `, [opts.sourceId ?? null, managed, minPerBucket]);
  } catch (err) {
    return {
      phase: 'consolidate',
      status: 'fail',
      duration_ms: 0,
      summary: 'failed to scan unconsolidated facts',
      details: { error: err instanceof Error ? err.message : String(err) },
      error: {
        class: 'ConsolidateScanFailed',
        code: 'consolidate_scan_failed',
        message: err instanceof Error ? err.message : String(err),
      },
    };
  }

  const run = new ItemRun('consolidate.buckets');
  const bucketFailures: BucketFailure[] = [];
  const deferral = await DreamWriteDeferralPass.start(engine, 'consolidate', managed && !dryRun);
  let bucketsFailed = reconciledBucketFailures(deferral, bucketFailures);
  for (const b of buckets) {
    // #1972: bail at the top of the bucket loop on abort. Each prior bucket's
    // per-row INSERT/consolidate is already committed, so breaking returns a
    // valid partial envelope (the inner cluster loop is bounded at limit 100,
    // so no inner guard is needed).
    if (isAborted(opts.signal)) break;
    if (opts.yieldDuringPhase) {
      try { await opts.yieldDuringPhase(); } catch { /* keepalive errors non-fatal */ }
    }

    // One bucket the writer refuses (an entity or evidence page made private or deleted while its world facts
    // stay active) is counted and sampled; later buckets still consolidate. Abort, budget and lost-connection
    // errors stay run-scoped; an accepted write still publishing defers its source (W3), or ends the phase with deferral off.
    if (deferral?.hold(b.source_id)) continue;
    let maintenance: MaintenanceAuthority | null = null;
    try {
      maintenance = managed && !dryRun ? await maintenancePreflight(engine, b.source_id) : null;
      const candidates = await engine.listFactsByEntity(b.source_id, b.entity_slug, {
        activeOnly: true,
        unconsolidatedOnly: true,
        visibility: managed ? ['world'] : undefined,
        limit: 100,
        eligibility: {},
      });
      // Takes here are the user's own (holder 'self'): a claim the assistant or
      // a named third party asserted is never promoted into one.
      const unconsolidated = (await withoutReadersSourced(engine, candidates)).filter(f => (!managed || f.visibility === 'world') && (!f.attributed_to || f.attributed_to === 'user'));
      if (unconsolidated.length < minPerBucket) {
        bucketsSkipped += 1;
        continue;
      }

      // Age gate: oldest must be at least minOldestAgeMs old.
      const oldest = unconsolidated.reduce((min, f) =>
        f.valid_from.getTime() < min.valid_from.getTime() ? f : min,
      );
      if (Date.now() - oldest.valid_from.getTime() < minOldestAgeMs) {
        bucketsSkipped += 1;
        continue;
      }

      bucketsProcessed += 1;
      const claims = await loadClaimShapes(engine, unconsolidated.map(f => f.id));
      // #5575 ENG-3: a take's tier is the least trusted fact in its cluster, so
      // external facts cluster apart from owner facts and their takes are external.
      const taint = new Map((await readTaintInputs(engine, unconsolidated.map(f => ({ table: 'facts' as const, id: f.id })))).map(i => [Number(i.id), i]));
      const clusters = [false, true].flatMap(external =>
        clusterFacts(unconsolidated.filter(f => isExternalTier(taint.get(f.id)!.tier) === external), threshold, claims));

      // Resolve entity_slug → page_id. If page missing in this source, skip.
      const pageRows = await engine.executeRaw<{ id: number }>(
        `SELECT id FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL LIMIT 1`,
        [b.source_id, b.entity_slug],
      );
      if (pageRows.length === 0) continue;
      const pageId = pageRows[0].id;

      // Existing row_num max for this page → start appending after it.
      const rowMaxRows = await engine.executeRaw<{ max: number }>(
        `SELECT COALESCE(MAX(row_num), 0)::int AS max FROM takes WHERE page_id = $1`,
        [pageId],
      );
      let nextRowNum = (rowMaxRows[0]?.max ?? 0) + 1;
      const recordsTies = !dryRun && !maintenance;
      if (recordsTies) await clearConsolidationTies(engine, b.source_id, b.entity_slug);

      for (const cluster of clusters) {
        if (cluster.length < 2) continue;
        const tie = clusterTie(cluster, claims);
        if (tie) { clustersHeldSameInstant += 1; if (sameInstantClusters.length < SAME_INSTANT_SAMPLES) sameInstantClusters.push(heldCluster(b, tie)); if (recordsTies) await recordTie(engine, b, cluster, tie); continue; }
        if (await consolidationRestored(engine, b.source_id, b.entity_slug, cluster.map(f => f.id))) continue; // the operator reopened this chain
        // Take selection: pick the highest-confidence fact's text as the
        // take claim (v0.31 deterministic). v0.32 will swap to a Sonnet
        // synthesis pass.
        const best = cluster.reduce((a, b) => (b.confidence > a.confidence ? b : a));
        const avgWeight = cluster.reduce((s, f) => s + f.confidence, 0) / cluster.length;
        const sources = Array.from(new Set(cluster.map(c => c.source_session ?? c.source).filter(Boolean))).join(',');
        const sinceISO = cluster
          .map(c => c.valid_from)
          .reduce((min, d) => (d < min ? d : min))
          .toISOString()
          .slice(0, 10);
        const inputs = cluster.map(f => taint.get(f.id)!);
        const derivation = { inputs, trust: derivedWriteTrust({ channel: 'derive:consolidate', inputs }) };

        if (dryRun) {
          // Pretend we did it.
          takesWritten += 1;
          factsConsolidated += cluster.length;
          nextRowNum += 1;
          if (preview.length < DRY_RUN_CLUSTER_CAP) {
            const text = (f: FactRow) => (showPrivate || f.visibility === 'world' ? f.fact : null);
            preview.push({ source_id: b.source_id, entity_slug: b.entity_slug, claim: text(best),
              members: cluster.map(f => ({ id: f.id, visibility: f.visibility, fact: text(f) })) });
          }
          continue;
        }

        if (maintenance) {
          const receipt = await submitMaintenanceConsolidation(engine, maintenance, b.entity_slug, cluster,
            { claim: best.fact, weight: clamp01(avgWeight), source: sources.slice(0, 200), since: sinceISO },
            declareDerivation(derivation.trust, inputs));
          factsConsolidated += Number(receipt.facts_consolidated ?? 0);
          takesWritten += Number(receipt.takes_written ?? 0);
          if (receipt.reason === 'retired_take') clustersSkippedRetired++;
          if (receipt.reason === 'same_instant_typed_claims') clustersHeldSameInstant++;
          continue;
        }

        // v0.35.4 (D-CDX-4) — semantic upsert. The full dream cycle runs
        // `extract_facts` BEFORE `consolidate`; `extract_facts` hard-deletes
        // and re-inserts page facts via deleteFactsForPage + insertFacts,
        // which clears `consolidated_at` on every fact. Without this lookup,
        // a second cycle run would re-INSERT a duplicate take via
        // `MAX(row_num)+1`, silently poisoning trajectory + scorecard data.
        // Match on (page_id, claim) restricted to the rows this phase authors.
        //
        // `since_date` is NOT part of the identity: it is derived from
        // MIN(valid_from) of the cluster, so it describes the fact rows that
        // happen to exist right now. When extract_facts re-inserts a claim with
        // a fresh `valid_from` (the common case: an LLM extraction that defaults
        // valid_from to now()), keying on since_date makes this lookup miss and
        // the upsert degrades into the duplicate-INSERT path this block exists
        // to prevent.
        //
        // `kind = 'fact' AND holder = 'self'` mirrors the INSERT below, so the
        // lookup can only ever match a take this phase wrote. Without it the
        // widened match reaches human-authored takes that share the claim text
        // and the UPDATE below would overwrite their provenance.
        //
        // Deliberately NOT filtered on `active`: a superseded take still owns
        // this claim. Skipping it would send us down the INSERT path and
        // resurrect a claim the user retired. `ORDER BY id` keeps the choice
        // deterministic if a page already holds more than one matching row
        // (duplicates minted by the pre-fix since_date lookup).
        const existing = await engine.executeRaw<{ id: number; resolved_at: Date | null }>(
          `SELECT id, resolved_at FROM takes
           WHERE page_id = $1 AND claim = $2 AND kind = 'fact' AND holder = 'self'
           ORDER BY id
           LIMIT 1`,
          [pageId, best.fact],
        );

        // #5575: the take, its source refresh, the consolidation marks and the
        // valid_until writeback are one maintenance transaction at the cluster's
        // derived tier. A re-promotion changes no content column, so the take is
        // lowered explicitly to min(existing, cluster): an external cluster
        // cannot keep an owner take's tier. Edges: take -> every cluster fact.
        // Write gate on a new take: reject skips the cluster; hold records it for review and leaves the facts unconsolidated.
        const takeRow = { page_id: pageId, row_num: nextRowNum, claim: best.fact, kind: 'fact', holder: 'self', weight: clamp01(avgWeight),
          since_date: sinceISO, source: sources.slice(0, 200), active: true };
        const gate = existing.length ? null : await gateNewTake(engine, derivation, takeRow, { sourceId: b.source_id, slug: b.entity_slug },
          gateConfig ??= (await loadImportSanityConfig(engine)).writeGate ?? DEFAULT_WRITE_GATE_CONFIG);
        if (gate && gate.action !== 'insert') { if (gate.action === 'hold') clustersGateHeld++; else clustersGateRejected++; continue; }
        const written = await derivedMaintenanceTransaction<{ created: boolean; takeId: number | null } | null>(engine, derivation, async tx => {
          let takeId: number;
          let created = false;
          if (existing.length > 0) {
            // Re-promotion of a cluster we already wrote a take for. Refresh
            // the source-aggregation string (new fact rows may carry new
            // source_session values that the prior run didn't see); leave
            // row_num + weight untouched to keep the take's identity stable.
            takeId = existing[0].id;
            // A resolved take is immutable everywhere else (the engines throw
            // TAKE_RESOLVED_IMMUTABLE / TAKE_ALREADY_RESOLVED); this raw UPDATE
            // would bypass that guard. Reuse its id so the facts still consolidate
            // into it, but leave the row untouched.
            if (existing[0].resolved_at === null) {
              await tx.executeRaw(`UPDATE takes SET source = $1, updated_at = now() WHERE id = $2`, [sources.slice(0, 200), takeId]);
            }
          } else {
            const inserted = await tx.addTakesBatch([takeRow]);
            if (inserted < 1) return { result: null, rows: [] };
            const idRows = await tx.executeRaw<{ id: number }>(
              `SELECT id FROM takes WHERE page_id = $1 AND row_num = $2`,
              [pageId, nextRowNum],
            );
            if (idRows.length === 0) return { result: { created: true, takeId: null }, rows: [] };
            takeId = idRows[0].id;
            created = true;
            if (gate) await recordFlaggedRow(tx, gate, { table: 'takes', id: takeId, sourceId: b.source_id });
          }

          await closeCluster(tx, b, cluster, takeId);
          return { result: { created, takeId }, rows: [{ table: 'takes' as const, id: Number(takeId), sourceId: b.source_id }] };
        });
        if (written?.created) nextRowNum += 1;
        if (!written?.takeId) continue;
        if (written.created) takesWritten += 1;
        factsConsolidated += cluster.length;
      }
      run.succeed();
    } catch (error) {
      if (classifyItemFailure(error).scope === 'pending') { if (maintenance && await deferral?.defer(error, { sourceId: b.source_id, subject: `${b.source_id}/${b.entity_slug}`, principal: maintenance.writer.principal })) continue; throw error; }
      const failure = run.fail(`${b.source_id}:${b.entity_slug}`, error);
      bucketsFailed += 1;
      sampleBucketFailure(bucketFailures, { source_id: b.source_id, entity_slug: b.entity_slug, code: failure.code, message: failure.message.slice(0, 300), fix: rerunFix(b.source_id) });
    }
  }

  return {
    phase: 'consolidate',
    status: bucketsFailed > 0 || clustersHeldSameInstant > 0 || deferral?.skipped ? 'warn' : factsConsolidated === 0 && clustersSkippedRetired > 0 ? 'skipped' : 'ok',
    duration_ms: 0,
    summary: (dryRun
      ? `(dry-run) would promote ${factsConsolidated} facts into ${takesWritten} takes across ${bucketsProcessed} buckets`
      : `promoted ${factsConsolidated} facts into ${takesWritten} takes across ${bucketsProcessed} buckets` +
        (clustersSkippedRetired ? `; skipped ${clustersSkippedRetired} clusters with retired takes` : '') +
        (clustersGateHeld ? `; held ${clustersGateHeld} takes for write-gate review` : '') +
        (clustersGateRejected ? `; write gate rejected ${clustersGateRejected} takes` : '') +
        (bucketsFailed ? `; ${bucketsFailed} bucket(s) failed (details.bucket_failures)` : '') + (deferral?.skipped ? `; deferred ${deferral.skipped} bucket(s): an accepted write is still publishing (details.write_deferral)` : '')) +
      (clustersHeldSameInstant ? `; held ${clustersHeldSameInstant} cluster(s) whose typed claims state one metric for the same effective time (details.same_instant_clusters)` : ''),
    details: {
      dryRun,
      cluster_threshold: threshold,
      untyped_cluster_threshold: untypedClusterThreshold(threshold),
      ...(thresholdInvalid !== undefined ? { cluster_threshold_invalid: thresholdInvalid } : {}),
      ...(dryRun ? { clusters: preview } : {}),
      facts_consolidated: factsConsolidated,
      takes_written: takesWritten,
      buckets_processed: bucketsProcessed,
      buckets_skipped: bucketsSkipped,
      clusters_skipped_retired: clustersSkippedRetired,
      clusters_gate_held: clustersGateHeld,
      clusters_gate_rejected: clustersGateRejected,
      clusters_held_same_instant: clustersHeldSameInstant,
      same_instant_clusters: sameInstantClusters,
      buckets_failed: bucketsFailed,
      bucket_failures: bucketFailures, ...deferral?.detailFields('buckets_deferred'),
      ...(factsConsolidated === 0 && clustersSkippedRetired > 0 ? { reason: 'retired_take' } : {}),
    },
  };
}

/**
 * #5575 write gate on one new unmanaged take, assessed at its cluster's
 * derived tier (the gate never assesses operator_curated or above). A hold is
 * recorded here, in an attributed transaction at that tier; the caller
 * inserts on `insert` (recording a flag receipt) and skips otherwise.
 */
async function gateNewTake(engine: BrainEngine, derivation: { trust: WriteTrust; inputs: readonly TaintInput[] },
  takeRow: { claim: string; source: string } & Record<string, unknown>, at: { sourceId: string; slug: string }, cfg: WriteGateConfig): Promise<GatedRowDecision> {
  const gate = decideTakeWrite({ claim: takeRow.claim, source: takeRow.source }, { ...at, payload: takeRow,
    input: { tier: derivation.trust.tier, origin: derivation.trust.origin, requestId: null }, cfg });
  if (gate.action === 'hold') await derivedMaintenanceTransaction(engine, derivation, async tx => ({ result: await recordWriteGateHold(tx, gate.hold!), rows: [] }));
  return gate;
}

/** One cluster held because two or more of its typed claims tie at one effective instant (`details.same_instant_clusters`). */
export interface SameInstantCluster {
  source_id: string;
  entity_slug: string;
  metric: string;
  unit: string | null;
  period: string | null;
  valid_from: string;
  fact_ids: number[];
  fix: Action;
}
const SAME_INSTANT_SAMPLES = 20;

function clusterTie(cluster: FactRow[], claims: Map<number, ClaimShape>): SameInstantTie | undefined {
  return sameInstantTies(cluster.map(f => ({ id: f.id, at: f.valid_from.getTime(), metric: claims.get(f.id)?.metric ?? null,
    unit: claims.get(f.id)?.unit ?? null, period: claims.get(f.id)?.period ?? null })))[0];
}

/**
 * Marks the cluster consolidated into `takeId` and closes its chain, recording each member's before-state as the cluster's
 * consolidation generation first (GBRA-161 PR 2), so `gbrain repair generations --kind consolidation` can reopen it.
 */
async function closeCluster(tx: BrainEngine, bucket: { source_id: string; entity_slug: string }, cluster: FactRow[], takeId: number): Promise<void> {
  const before = await factPreimages(tx, bucket.source_id, cluster.map(f => f.id));
  for (const f of cluster) await tx.consolidateFact(f.id, takeId);
  await writeValidUntilChain(tx, cluster);
  await recordConsolidationGeneration(tx, bucket.source_id, bucket.entity_slug, before);
}

/** #6023 (GBRA-161 PR 2): records a tied cluster as a held consolidation generation (`tie`); nothing is written to its facts. */
async function recordTie(engine: BrainEngine, bucket: { source_id: string; entity_slug: string }, cluster: FactRow[], tie: SameInstantTie): Promise<void> {
  await engine.transaction(tx => holdConsolidationTie(tx, bucket.source_id, bucket.entity_slug, cluster.map(f => f.id),
    { entity_slug: bucket.entity_slug, tied: tie.ids, valid_from: new Date(tie.at).toISOString(), metric: tie.metric }));
}

function heldCluster(bucket: { source_id: string; entity_slug: string }, tie: SameInstantTie): SameInstantCluster {
  const validFrom = new Date(tie.at).toISOString();
  const claim = [tie.metric, tie.unit, tie.period].filter(Boolean).join(', ');
  return { source_id: bucket.source_id, entity_slug: bucket.entity_slug, metric: tie.metric, unit: tie.unit, period: tie.period, valid_from: validFrom,
    fact_ids: tie.ids, fix: { ask_user: true, argv: ['gbrain', 'forget', '<FACT_ID>'], consent: [], actor: 'agent', requires_exclusive: false,
      inputs: [{ name: 'FACT_ID', how: `One of facts ${tie.ids.join(', ')}: the statement the user says is no longer current; for exact duplicates, any one of them.` }],
      why: 'Consolidation cannot order statements of one metric at one effective time, so it left the cluster unchanged; forgetting the stale one lets the next consolidate run promote the rest.',
      user_message: `Facts ${tie.ids.join(', ')} on ${bucket.entity_slug} state ${claim} for the same effective time (${validFrom}), and gbrain cannot tell which one is current, so it left them unconsolidated. Which one should stay? The other is forgotten, and the next consolidate run promotes the cluster.` } };
}

/** The typed-claim columns of a fact (FactRow does not carry them). */
export interface ClaimShape {
  metric: string | null;
  unit: string | null;
  period: string | null;
}

async function loadClaimShapes(engine: BrainEngine, ids: number[]): Promise<Map<number, ClaimShape>> {
  const rows = ids.length === 0 ? [] : await engine.executeRaw<{
    id: number; claim_metric: string | null; claim_unit: string | null; claim_period: string | null;
  }>(`SELECT id, claim_metric, claim_unit, claim_period FROM facts WHERE id = ANY($1::bigint[])`, [ids]);
  return new Map(rows.map(r => [Number(r.id), { metric: r.claim_metric, unit: r.claim_unit, period: r.claim_period }]));
}

/**
 * #5363: two facts may share a take only when their typed claims agree.
 * Untyped facts (no metric) cluster with untyped facts; a typed fact
 * clusters only with facts of the same metric, unit and period (a monthly
 * and an annual MRR, or USD and EUR, never merge however close their text
 * embeddings are). A fact whose claim columns are unknown is treated as
 * untyped.
 */
export function claimsCompatible(a: ClaimShape | undefined, b: ClaimShape | undefined): boolean {
  const metricA = a?.metric ?? null;
  const metricB = b?.metric ?? null;
  if (metricA === null || metricB === null) return metricA === metricB;
  return metricA === metricB && (a?.unit ?? null) === (b?.unit ?? null) && (a?.period ?? null) === (b?.period ?? null);
}

/**
 * #5363: the knob was unreachable (cycle.ts never passed it); the phase reads
 * its own config key. A malformed stored value keeps the default and is
 * reported in details (`cluster_threshold_invalid`) instead of failing the cycle.
 */
async function resolveClusterThreshold(
  engine: BrainEngine,
  override: number | undefined,
): Promise<{ threshold: number; thresholdInvalid?: string }> {
  if (override !== undefined) return { threshold: override };
  const configured = await engine.getConfig(CLUSTER_THRESHOLD_KEY);
  if (configured == null || configured.trim() === '') return { threshold: DEFAULT_CLUSTER_THRESHOLD };
  try { return { threshold: parseClusterThreshold(configured) }; }
  catch { return { threshold: DEFAULT_CLUSTER_THRESHOLD, thresholdInvalid: configured }; }
}

/** One cluster a dry run would promote (`details.clusters`, capped at DRY_RUN_CLUSTER_CAP). */
export interface ConsolidateClusterPreview {
  source_id: string;
  entity_slug: string;
  /** The would-be take claim; null when the fact is private and the caller is not the local CLI. */
  claim: string | null;
  members: Array<{ id: number; visibility: string; fact: string | null }>;
}
export const DRY_RUN_CLUSTER_CAP = 50;

/**
 * #6023: untyped facts have no metric/unit/period guard, so two event facts
 * that differ only in an identifier, date or amount could cluster on a 0.85
 * embedding and the writeback would expire the older one. They cluster only
 * at the explicit-duplicate bar (never below the configured threshold).
 */
export function untypedClusterThreshold(threshold: number): number {
  return Math.max(threshold, EXPLICIT_DUPLICATE_THRESHOLD);
}

/**
 * Greedy cosine clustering. Iterate facts sorted by valid_from DESC; each
 * fact joins the first cluster whose centroid (the first member, for
 * simplicity) is within `threshold` cosine and whose typed claim is
 * compatible (#5363). Otherwise starts a new cluster.
 *
 * #6023: an untyped pair additionally needs cosine ≥ the explicit-duplicate
 * bar (`untypedClusterThreshold`) and must not diverge in a negation, number,
 * date or amount token (`claimsDiverge`): "Invoice A due Monday" and
 * "Invoice B due Tuesday" stay two facts. Typed facts keep the configured
 * threshold and no token guard, because a metric series is expected to
 * differ in its values (that is the trajectory the writeback records).
 *
 * Facts with no embedding cluster on their own (single-element cluster);
 * the consolidate phase only writes takes from clusters of size ≥ 2, so
 * no-embedding singletons sit out the cycle. v0.32+ fact-extraction
 * pipeline ensures embeddings are computed at insertFact time.
 */
export function clusterFacts(facts: FactRow[], threshold: number, claims: Map<number, ClaimShape> = new Map()): FactRow[][] {
  const sorted = [...facts].sort((a, b) => b.valid_from.getTime() - a.valid_from.getTime());
  const clusters: FactRow[][] = [];
  for (const f of sorted) {
    if (!f.embedding || !f.embedding_model || f.embedded_text_hash !== createHash('md5').update(f.fact).digest('hex')) {
      clusters.push([f]);
      continue;
    }
    let placed = false;
    for (const c of clusters) {
      const head = c[0];
      if (!head.embedding || f.embedding_model !== head.embedding_model
        || head.embedded_text_hash !== createHash('md5').update(head.fact).digest('hex')
        || f.embedding.length !== head.embedding.length
        || !claimsCompatible(claims.get(f.id), claims.get(head.id))) continue;
      const untyped = (claims.get(f.id)?.metric ?? null) === null;
      if (untyped && claimsDiverge(f.fact, head.fact)) continue;
      if (cosineSimilarity(f.embedding, head.embedding) >= (untyped ? untypedClusterThreshold(threshold) : threshold)) {
        c.push(f);
        placed = true;
        break;
      }
    }
    if (!placed) clusters.push([f]);
  }
  return clusters;
}

function clamp01(x: number): number {
  if (!Number.isFinite(x)) return 0.5;
  if (x < 0) return 0;
  if (x > 1) return 1;
  return x;
}