Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.31 — Dream-cycle `consolidate` phase: facts → takes promotion.
*
* Per /plan-eng-review Phase 5:
*
* For each (source_id, entity_slug) bucket of unconsolidated active facts
* the user asserted (or with no recorded speaker):
* 1. Skip if count < 3 OR oldest fact age < 24h.
* 2. Cluster by embedding cosine — greedy threshold 0.85.
* 3. For each cluster ≥ 2: pick the highest-confidence fact's text as
* the take claim (v0.31 ships without LLM synthesis to keep the
* cycle deterministic; see TODO at the bottom for the v0.32 Sonnet
* rewrite).
* 4. Resolve entity_slug → pages.slug. If the page is missing, skip
* this cluster (no auto-page-creation in v0.31; the take needs a
* home).
* 5. INSERT into takes(kind='fact', holder='self', source=concatenated
* source_sessions). row_num = MAX existing for the page + 1.
* 6. UPDATE contributing facts: consolidated_at = now() +
* consolidated_into = takes.id. NEVER DELETE.
*
* The phase's totals contribute to the runCycle CycleReport via
* extractTotals (cycle.ts) — facts_consolidated + takes_written.
*/
import type { Action } from '../../agent-output.ts';
import type { BrainEngine, FactRow } from '../../engine.ts';
import type { PhaseResult } from '../../cycle.ts';
import { cosineSimilarity } from '../../facts/classify.ts';
import { claimsDiverge, EXPLICIT_DUPLICATE_THRESHOLD } from '../../facts/capture-dedup.ts';
import { sameInstantTies, type SameInstantTie } from '../../facts/claim-ties.ts';
import { createHash } from 'node:crypto';
import { isAborted } from '../../abort-check.ts';
import { classifyItemFailure, ItemRun } from '../../item-failure.ts';
import { consolidationCloses, maintenancePreflight, submitMaintenanceConsolidation, type MaintenanceAuthority } from '../../persistence/prepared-maintenance.ts';
import { clearConsolidationTies, consolidationRestored, factPreimages, holdConsolidationTie, recordConsolidationGeneration } from '../../generations/adapters/fact-rows.ts';
import { managedPersistenceEnabled } from '../../persistence/ownership.ts';
import { DreamWriteDeferralPass } from '../dream-write-deferral.ts';
import { declareDerivation, derivedMaintenanceTransaction, derivedWriteTrust, isExternalTier, readTaintInputs } from '../../trust/taint.ts';
import { DEFAULT_WRITE_GATE_CONFIG, type WriteGateConfig } from '../../write-gate.ts';
import { loadImportSanityConfig } from '../../import-screen.ts';
import { decideTakeWrite, recordFlaggedRow, recordWriteGateHold, type GatedRowDecision } from '../../write-gate-store.ts';
import type { TaintInput, WriteTrust } from '../../trust/tier.ts';
import { readersSourcedFactSql } from '../../access/member-facts.ts';
export interface ConsolidatePhaseOpts {
dryRun?: boolean;
/** In-phase keepalive callback. Awaited between buckets. */
yieldDuringPhase?: () => Promise<void>;
/**
* #1972: cooperative-abort signal. Checked at the top of the bucket loop so a
* long consolidate relinquishes its worker slot well under the 30s
* force-evict instead of running to completion after cancellation.
*/
signal?: AbortSignal;
/**
* Cosine cluster threshold for typed (metric) facts. Overrides
* `cycle.consolidate.cluster_threshold`; default 0.85. Untyped facts cluster
* at `max(threshold, EXPLICIT_DUPLICATE_THRESHOLD)` (#6023).
*/
clusterThreshold?: number;
/**
* Trust of the caller reading a dry-run preview. Fail-closed: only
* `remote === false` (the local CLI) sees the text of private facts in
* `details.clusters`; anything else sees their ids and visibility only.
*/
remote?: boolean;
/** Minimum facts per (source, entity) bucket before consolidation. Default 3. */
minFactsPerBucket?: number;
/** Minimum age (ms) of the OLDEST fact in a bucket before consolidation. Default 24h. */
minOldestAgeMs?: number;
sourceId?: string;
/** `gbrain dream --phase consolidate --once`: run even when `cycle.consolidate.enabled` is false. */
once?: boolean;
}
/**
* #6667: `cycle.consolidate.enabled` (default on) is the off switch for this
* phase, which promotes unreviewed facts into takes. Only an explicit falsy
* value skips it; unset, a truthy value and a config read failure keep it on.
*/
export const CONSOLIDATE_ENABLED_KEY = 'cycle.consolidate.enabled';
async function disabledSkip(engine: BrainEngine, opts: ConsolidatePhaseOpts): Promise<PhaseResult | null> {
if (opts.once || !/^\s*(false|0|off|no)\s*$/i.test(await engine.getConfig(CONSOLIDATE_ENABLED_KEY).catch(() => null) ?? '')) return null;
return { phase: 'consolidate', status: 'skipped', duration_ms: 0, summary: `consolidate skipped: ${CONSOLIDATE_ENABLED_KEY}=false`,
details: { reason: 'disabled', enable_hint: `gbrain config set ${CONSOLIDATE_ENABLED_KEY} true` } };
}
/** At most this many failed buckets are listed in `details.bucket_failures`; `buckets_failed` counts them all. */
const BUCKET_FAILURE_SAMPLES = 10;
type BucketFailure = { source_id: string; entity_slug: string; code: string; message: string; fix: Action };
const rerunFix = (sourceId: string): Action => ({ argv: ['gbrain', 'dream', '--phase', 'consolidate', '--source', sourceId], consent: [], actor: 'agent',
requires_exclusive: false, why: 'Reruns consolidate for this source; the bucket consolidates once its entity and evidence pages are readable again.' });
/**
* W3 (#7587): consolidations an earlier pass left pending that ended failed, as bucket failures (samples pushed onto
* `failures`); returns how many. A source whose receipt is still pending submits nothing this pass (`hold`), and a
* write still pending after its wait defers the rest of its source (`defer`); deferral off restores the failing phase.
*/
function reconciledBucketFailures(deferral: DreamWriteDeferralPass | null, failures: BucketFailure[]): number {
const failed = deferral?.reconcile.failed ?? [];
for (const entry of failed.slice(0, Math.max(0, BUCKET_FAILURE_SAMPLES - failures.length))) {
failures.push({ source_id: entry.source_id, entity_slug: entry.subject.slice(entry.source_id.length + 1), code: entry.code,
message: `The consolidation an earlier pass left pending (request_id ${entry.request_id}) ended with ${entry.code}: ${entry.message}`.slice(0, 300), fix: rerunFix(entry.source_id) });
}
return failed.length;
}
/** Samples a failed bucket into `details.bucket_failures`, at most BUCKET_FAILURE_SAMPLES of them. */
function sampleBucketFailure(failures: BucketFailure[], failure: BucketFailure): void {
if (failures.length < BUCKET_FAILURE_SAMPLES) failures.push(failure);
}
export const CLUSTER_THRESHOLD_KEY = 'cycle.consolidate.cluster_threshold';
export const DEFAULT_CLUSTER_THRESHOLD = 0.85;
/**
* v0.35.4 (D-CDX-4 part 2) — chronological valid_until writeback.
* Sort the cluster by (valid_from ASC, id ASC); stamp each fact's
* valid_until = the valid_from of the next fact that says something
* else, by claim text and typed value (consolidationCloses). A restatement of the same claim does not
* end the fact before it; the newest value keeps valid_until = NULL. This makes the facts
* table a proper bitemporal record without the contradiction probe
* having to mutate it (preserves auto-supersession.ts:4 invariant —
* see also R8 test guard).
*
* Idempotent: re-running on the same cluster produces the same
* chronological order and the same valid_until values. No-op if
* valid_until is already correct.
*/
async function writeValidUntilChain(tx: BrainEngine, cluster: FactRow[]): Promise<void> {
const chronological = [...cluster].sort((a, b) => {
const t = a.valid_from.getTime() - b.valid_from.getTime();
if (t !== 0) return t;
return a.id - b.id;
});
const values = new Map((await tx.executeRaw<{ id: number; claim_value: number | string | null }>(
'SELECT id, claim_value FROM facts WHERE id = ANY($1::bigint[])', [cluster.map(f => f.id)])).map(r => [Number(r.id), r.claim_value]));
for (const { member: older, next: newer } of consolidationCloses(chronological, f => ({ text: f.fact, value: values.get(f.id) }))) {
await tx.executeRaw(
// Only UPDATE when the new value would actually change. Avoids
// touching updated_at on no-op rewrites and keeps idempotency
// observable in the DB (zero affected rows on stable re-run).
`UPDATE facts
SET valid_until = $1
WHERE id = $2
AND (valid_until IS DISTINCT FROM $1)`,
[newer.valid_from, older.id],
);
}
}
/**
* #5363: parse `cycle.consolidate.cluster_threshold`. A cosine similarity in
* (0, 1]; lower values merge more facts into one take. Throws on anything
* else so `gbrain config set` can refuse it.
*/
export function parseClusterThreshold(value: string): number {
const n = Number(value.trim());
if (value.trim() === '' || !Number.isFinite(n) || n <= 0 || n > 1) {
throw new Error(`${CLUSTER_THRESHOLD_KEY} must be a number greater than 0 and at most 1 (default ${DEFAULT_CLUSTER_THRESHOLD}); got '${value}'.`);
}
return n;
}
/** GST-359 C2: `facts` without those on a page that names readers (a member page): they never become a take on the entity page. */
async function withoutReadersSourced(engine: BrainEngine, facts: FactRow[]): Promise<FactRow[]> {
const rows = await engine.executeRaw<{ id: number }>(`SELECT id FROM facts WHERE id = ANY($1::bigint[]) AND ${readersSourcedFactSql('facts')}`, [facts.map(f => f.id)]);
const restricted = new Set(rows.map(row => Number(row.id)));
return facts.filter(f => !restricted.has(f.id));
}
export async function runPhaseConsolidate(
engine: BrainEngine,
opts: ConsolidatePhaseOpts = {},
): Promise<PhaseResult> {
const disabled = await disabledSkip(engine, opts);
if (disabled) return disabled;
const dryRun = opts.dryRun === true;
const managed = await managedPersistenceEnabled(engine);
const { threshold, thresholdInvalid } = await resolveClusterThreshold(engine, opts.clusterThreshold);
const minPerBucket = opts.minFactsPerBucket ?? 3;
const minOldestAgeMs = opts.minOldestAgeMs ?? 24 * 60 * 60 * 1000;
let factsConsolidated = 0;
let takesWritten = 0;
let bucketsProcessed = 0;
let bucketsSkipped = 0;
let clustersSkippedRetired = 0;
let clustersGateHeld = 0;
let clustersGateRejected = 0;
let clustersHeldSameInstant = 0;
const sameInstantClusters: SameInstantCluster[] = [];
// #6023: a dry run lists the clusters it would write so an operator can
// inspect them before `--yes`; capped so the envelope stays bounded.
const preview: ConsolidateClusterPreview[] = [];
const showPrivate = opts.remote === false;
// #5575 write gate: unmanaged takes are gated here (managed ones in takes-prepare); read once per run.
let gateConfig: WriteGateConfig | undefined;
// Pull every (source_id, entity_slug) bucket of unconsolidated facts whose entity is a live page (a take needs one, and
// GBRA-574: an unbacked name key has none, so its bucket is never read). Uses the partial idx_facts_unconsolidated index.
let buckets: Array<{ source_id: string; entity_slug: string; count: number }>;
try {
buckets = await engine.executeRaw<{
source_id: string; entity_slug: string; count: number;
}>(`
SELECT source_id, entity_slug, COUNT(*)::int AS count
FROM facts
WHERE consolidated_at IS NULL
AND expired_at IS NULL
AND (valid_until IS NULL OR valid_until > now())
AND entity_slug IS NOT NULL
AND EXISTS (SELECT 1 FROM pages bucket_page WHERE bucket_page.source_id = facts.source_id AND bucket_page.slug = facts.entity_slug AND bucket_page.deleted_at IS NULL)
AND (attributed_to IS NULL OR attributed_to = 'user')
AND ($2::boolean=false OR visibility='world')
AND ($1::text IS NULL OR source_id=$1) AND NOT ${readersSourcedFactSql('facts')}
GROUP BY source_id, entity_slug
HAVING COUNT(*) >= $3
`, [opts.sourceId ?? null, managed, minPerBucket]);
} catch (err) {
return {
phase: 'consolidate',
status: 'fail',
duration_ms: 0,
summary: 'failed to scan unconsolidated facts',
details: { error: err instanceof Error ? err.message : String(err) },
error: {
class: 'ConsolidateScanFailed',
code: 'consolidate_scan_failed',
message: err instanceof Error ? err.message : String(err),
},
};
}
const run = new ItemRun('consolidate.buckets');
const bucketFailures: BucketFailure[] = [];
const deferral = await DreamWriteDeferralPass.start(engine, 'consolidate', managed && !dryRun);
let bucketsFailed = reconciledBucketFailures(deferral, bucketFailures);
for (const b of buckets) {
// #1972: bail at the top of the bucket loop on abort. Each prior bucket's
// per-row INSERT/consolidate is already committed, so breaking returns a
// valid partial envelope (the inner cluster loop is bounded at limit 100,
// so no inner guard is needed).
if (isAborted(opts.signal)) break;
if (opts.yieldDuringPhase) {
try { await opts.yieldDuringPhase(); } catch { /* keepalive errors non-fatal */ }
}
// One bucket the writer refuses (an entity or evidence page made private or deleted while its world facts
// stay active) is counted and sampled; later buckets still consolidate. Abort, budget and lost-connection
// errors stay run-scoped; an accepted write still publishing defers its source (W3), or ends the phase with deferral off.
if (deferral?.hold(b.source_id)) continue;
let maintenance: MaintenanceAuthority | null = null;
try {
maintenance = managed && !dryRun ? await maintenancePreflight(engine, b.source_id) : null;
const candidates = await engine.listFactsByEntity(b.source_id, b.entity_slug, {
activeOnly: true,
unconsolidatedOnly: true,
visibility: managed ? ['world'] : undefined,
limit: 100,
eligibility: {},
});
// Takes here are the user's own (holder 'self'): a claim the assistant or
// a named third party asserted is never promoted into one.
const unconsolidated = (await withoutReadersSourced(engine, candidates)).filter(f => (!managed || f.visibility === 'world') && (!f.attributed_to || f.attributed_to === 'user'));
if (unconsolidated.length < minPerBucket) {
bucketsSkipped += 1;
continue;
}
// Age gate: oldest must be at least minOldestAgeMs old.
const oldest = unconsolidated.reduce((min, f) =>
f.valid_from.getTime() < min.valid_from.getTime() ? f : min,
);
if (Date.now() - oldest.valid_from.getTime() < minOldestAgeMs) {
bucketsSkipped += 1;
continue;
}
bucketsProcessed += 1;
const claims = await loadClaimShapes(engine, unconsolidated.map(f => f.id));
// #5575 ENG-3: a take's tier is the least trusted fact in its cluster, so
// external facts cluster apart from owner facts and their takes are external.
const taint = new Map((await readTaintInputs(engine, unconsolidated.map(f => ({ table: 'facts' as const, id: f.id })))).map(i => [Number(i.id), i]));
const clusters = [false, true].flatMap(external =>
clusterFacts(unconsolidated.filter(f => isExternalTier(taint.get(f.id)!.tier) === external), threshold, claims));
// Resolve entity_slug → page_id. If page missing in this source, skip.
const pageRows = await engine.executeRaw<{ id: number }>(
`SELECT id FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL LIMIT 1`,
[b.source_id, b.entity_slug],
);
if (pageRows.length === 0) continue;
const pageId = pageRows[0].id;
// Existing row_num max for this page → start appending after it.
const rowMaxRows = await engine.executeRaw<{ max: number }>(
`SELECT COALESCE(MAX(row_num), 0)::int AS max FROM takes WHERE page_id = $1`,
[pageId],
);
let nextRowNum = (rowMaxRows[0]?.max ?? 0) + 1;
const recordsTies = !dryRun && !maintenance;
if (recordsTies) await clearConsolidationTies(engine, b.source_id, b.entity_slug);
for (const cluster of clusters) {
if (cluster.length < 2) continue;
const tie = clusterTie(cluster, claims);
if (tie) { clustersHeldSameInstant += 1; if (sameInstantClusters.length < SAME_INSTANT_SAMPLES) sameInstantClusters.push(heldCluster(b, tie)); if (recordsTies) await recordTie(engine, b, cluster, tie); continue; }
if (await consolidationRestored(engine, b.source_id, b.entity_slug, cluster.map(f => f.id))) continue; // the operator reopened this chain
// Take selection: pick the highest-confidence fact's text as the
// take claim (v0.31 deterministic). v0.32 will swap to a Sonnet
// synthesis pass.
const best = cluster.reduce((a, b) => (b.confidence > a.confidence ? b : a));
const avgWeight = cluster.reduce((s, f) => s + f.confidence, 0) / cluster.length;
const sources = Array.from(new Set(cluster.map(c => c.source_session ?? c.source).filter(Boolean))).join(',');
const sinceISO = cluster
.map(c => c.valid_from)
.reduce((min, d) => (d < min ? d : min))
.toISOString()
.slice(0, 10);
const inputs = cluster.map(f => taint.get(f.id)!);
const derivation = { inputs, trust: derivedWriteTrust({ channel: 'derive:consolidate', inputs }) };
if (dryRun) {
// Pretend we did it.
takesWritten += 1;
factsConsolidated += cluster.length;
nextRowNum += 1;
if (preview.length < DRY_RUN_CLUSTER_CAP) {
const text = (f: FactRow) => (showPrivate || f.visibility === 'world' ? f.fact : null);
preview.push({ source_id: b.source_id, entity_slug: b.entity_slug, claim: text(best),
members: cluster.map(f => ({ id: f.id, visibility: f.visibility, fact: text(f) })) });
}
continue;
}
if (maintenance) {
const receipt = await submitMaintenanceConsolidation(engine, maintenance, b.entity_slug, cluster,
{ claim: best.fact, weight: clamp01(avgWeight), source: sources.slice(0, 200), since: sinceISO },
declareDerivation(derivation.trust, inputs));
factsConsolidated += Number(receipt.facts_consolidated ?? 0);
takesWritten += Number(receipt.takes_written ?? 0);
if (receipt.reason === 'retired_take') clustersSkippedRetired++;
if (receipt.reason === 'same_instant_typed_claims') clustersHeldSameInstant++;
continue;
}
// v0.35.4 (D-CDX-4) — semantic upsert. The full dream cycle runs
// `extract_facts` BEFORE `consolidate`; `extract_facts` hard-deletes
// and re-inserts page facts via deleteFactsForPage + insertFacts,
// which clears `consolidated_at` on every fact. Without this lookup,
// a second cycle run would re-INSERT a duplicate take via
// `MAX(row_num)+1`, silently poisoning trajectory + scorecard data.
// Match on (page_id, claim) restricted to the rows this phase authors.
//
// `since_date` is NOT part of the identity: it is derived from
// MIN(valid_from) of the cluster, so it describes the fact rows that
// happen to exist right now. When extract_facts re-inserts a claim with
// a fresh `valid_from` (the common case: an LLM extraction that defaults
// valid_from to now()), keying on since_date makes this lookup miss and
// the upsert degrades into the duplicate-INSERT path this block exists
// to prevent.
//
// `kind = 'fact' AND holder = 'self'` mirrors the INSERT below, so the
// lookup can only ever match a take this phase wrote. Without it the
// widened match reaches human-authored takes that share the claim text
// and the UPDATE below would overwrite their provenance.
//
// Deliberately NOT filtered on `active`: a superseded take still owns
// this claim. Skipping it would send us down the INSERT path and
// resurrect a claim the user retired. `ORDER BY id` keeps the choice
// deterministic if a page already holds more than one matching row
// (duplicates minted by the pre-fix since_date lookup).
const existing = await engine.executeRaw<{ id: number; resolved_at: Date | null }>(
`SELECT id, resolved_at FROM takes
WHERE page_id = $1 AND claim = $2 AND kind = 'fact' AND holder = 'self'
ORDER BY id
LIMIT 1`,
[pageId, best.fact],
);
// #5575: the take, its source refresh, the consolidation marks and the
// valid_until writeback are one maintenance transaction at the cluster's
// derived tier. A re-promotion changes no content column, so the take is
// lowered explicitly to min(existing, cluster): an external cluster
// cannot keep an owner take's tier. Edges: take -> every cluster fact.
// Write gate on a new take: reject skips the cluster; hold records it for review and leaves the facts unconsolidated.
const takeRow = { page_id: pageId, row_num: nextRowNum, claim: best.fact, kind: 'fact', holder: 'self', weight: clamp01(avgWeight),
since_date: sinceISO, source: sources.slice(0, 200), active: true };
const gate = existing.length ? null : await gateNewTake(engine, derivation, takeRow, { sourceId: b.source_id, slug: b.entity_slug },
gateConfig ??= (await loadImportSanityConfig(engine)).writeGate ?? DEFAULT_WRITE_GATE_CONFIG);
if (gate && gate.action !== 'insert') { if (gate.action === 'hold') clustersGateHeld++; else clustersGateRejected++; continue; }
const written = await derivedMaintenanceTransaction<{ created: boolean; takeId: number | null } | null>(engine, derivation, async tx => {
let takeId: number;
let created = false;
if (existing.length > 0) {
// Re-promotion of a cluster we already wrote a take for. Refresh
// the source-aggregation string (new fact rows may carry new
// source_session values that the prior run didn't see); leave
// row_num + weight untouched to keep the take's identity stable.
takeId = existing[0].id;
// A resolved take is immutable everywhere else (the engines throw
// TAKE_RESOLVED_IMMUTABLE / TAKE_ALREADY_RESOLVED); this raw UPDATE
// would bypass that guard. Reuse its id so the facts still consolidate
// into it, but leave the row untouched.
if (existing[0].resolved_at === null) {
await tx.executeRaw(`UPDATE takes SET source = $1, updated_at = now() WHERE id = $2`, [sources.slice(0, 200), takeId]);
}
} else {
const inserted = await tx.addTakesBatch([takeRow]);
if (inserted < 1) return { result: null, rows: [] };
const idRows = await tx.executeRaw<{ id: number }>(
`SELECT id FROM takes WHERE page_id = $1 AND row_num = $2`,
[pageId, nextRowNum],
);
if (idRows.length === 0) return { result: { created: true, takeId: null }, rows: [] };
takeId = idRows[0].id;
created = true;
if (gate) await recordFlaggedRow(tx, gate, { table: 'takes', id: takeId, sourceId: b.source_id });
}
await closeCluster(tx, b, cluster, takeId);
return { result: { created, takeId }, rows: [{ table: 'takes' as const, id: Number(takeId), sourceId: b.source_id }] };
});
if (written?.created) nextRowNum += 1;
if (!written?.takeId) continue;
if (written.created) takesWritten += 1;
factsConsolidated += cluster.length;
}
run.succeed();
} catch (error) {
if (classifyItemFailure(error).scope === 'pending') { if (maintenance && await deferral?.defer(error, { sourceId: b.source_id, subject: `${b.source_id}/${b.entity_slug}`, principal: maintenance.writer.principal })) continue; throw error; }
const failure = run.fail(`${b.source_id}:${b.entity_slug}`, error);
bucketsFailed += 1;
sampleBucketFailure(bucketFailures, { source_id: b.source_id, entity_slug: b.entity_slug, code: failure.code, message: failure.message.slice(0, 300), fix: rerunFix(b.source_id) });
}
}
return {
phase: 'consolidate',
status: bucketsFailed > 0 || clustersHeldSameInstant > 0 || deferral?.skipped ? 'warn' : factsConsolidated === 0 && clustersSkippedRetired > 0 ? 'skipped' : 'ok',
duration_ms: 0,
summary: (dryRun
? `(dry-run) would promote ${factsConsolidated} facts into ${takesWritten} takes across ${bucketsProcessed} buckets`
: `promoted ${factsConsolidated} facts into ${takesWritten} takes across ${bucketsProcessed} buckets` +
(clustersSkippedRetired ? `; skipped ${clustersSkippedRetired} clusters with retired takes` : '') +
(clustersGateHeld ? `; held ${clustersGateHeld} takes for write-gate review` : '') +
(clustersGateRejected ? `; write gate rejected ${clustersGateRejected} takes` : '') +
(bucketsFailed ? `; ${bucketsFailed} bucket(s) failed (details.bucket_failures)` : '') + (deferral?.skipped ? `; deferred ${deferral.skipped} bucket(s): an accepted write is still publishing (details.write_deferral)` : '')) +
(clustersHeldSameInstant ? `; held ${clustersHeldSameInstant} cluster(s) whose typed claims state one metric for the same effective time (details.same_instant_clusters)` : ''),
details: {
dryRun,
cluster_threshold: threshold,
untyped_cluster_threshold: untypedClusterThreshold(threshold),
...(thresholdInvalid !== undefined ? { cluster_threshold_invalid: thresholdInvalid } : {}),
...(dryRun ? { clusters: preview } : {}),
facts_consolidated: factsConsolidated,
takes_written: takesWritten,
buckets_processed: bucketsProcessed,
buckets_skipped: bucketsSkipped,
clusters_skipped_retired: clustersSkippedRetired,
clusters_gate_held: clustersGateHeld,
clusters_gate_rejected: clustersGateRejected,
clusters_held_same_instant: clustersHeldSameInstant,
same_instant_clusters: sameInstantClusters,
buckets_failed: bucketsFailed,
bucket_failures: bucketFailures, ...deferral?.detailFields('buckets_deferred'),
...(factsConsolidated === 0 && clustersSkippedRetired > 0 ? { reason: 'retired_take' } : {}),
},
};
}
/**
* #5575 write gate on one new unmanaged take, assessed at its cluster's
* derived tier (the gate never assesses operator_curated or above). A hold is
* recorded here, in an attributed transaction at that tier; the caller
* inserts on `insert` (recording a flag receipt) and skips otherwise.
*/
async function gateNewTake(engine: BrainEngine, derivation: { trust: WriteTrust; inputs: readonly TaintInput[] },
takeRow: { claim: string; source: string } & Record<string, unknown>, at: { sourceId: string; slug: string }, cfg: WriteGateConfig): Promise<GatedRowDecision> {
const gate = decideTakeWrite({ claim: takeRow.claim, source: takeRow.source }, { ...at, payload: takeRow,
input: { tier: derivation.trust.tier, origin: derivation.trust.origin, requestId: null }, cfg });
if (gate.action === 'hold') await derivedMaintenanceTransaction(engine, derivation, async tx => ({ result: await recordWriteGateHold(tx, gate.hold!), rows: [] }));
return gate;
}
/** One cluster held because two or more of its typed claims tie at one effective instant (`details.same_instant_clusters`). */
export interface SameInstantCluster {
source_id: string;
entity_slug: string;
metric: string;
unit: string | null;
period: string | null;
valid_from: string;
fact_ids: number[];
fix: Action;
}
const SAME_INSTANT_SAMPLES = 20;
function clusterTie(cluster: FactRow[], claims: Map<number, ClaimShape>): SameInstantTie | undefined {
return sameInstantTies(cluster.map(f => ({ id: f.id, at: f.valid_from.getTime(), metric: claims.get(f.id)?.metric ?? null,
unit: claims.get(f.id)?.unit ?? null, period: claims.get(f.id)?.period ?? null })))[0];
}
/**
* Marks the cluster consolidated into `takeId` and closes its chain, recording each member's before-state as the cluster's
* consolidation generation first (GBRA-161 PR 2), so `gbrain repair generations --kind consolidation` can reopen it.
*/
async function closeCluster(tx: BrainEngine, bucket: { source_id: string; entity_slug: string }, cluster: FactRow[], takeId: number): Promise<void> {
const before = await factPreimages(tx, bucket.source_id, cluster.map(f => f.id));
for (const f of cluster) await tx.consolidateFact(f.id, takeId);
await writeValidUntilChain(tx, cluster);
await recordConsolidationGeneration(tx, bucket.source_id, bucket.entity_slug, before);
}
/** #6023 (GBRA-161 PR 2): records a tied cluster as a held consolidation generation (`tie`); nothing is written to its facts. */
async function recordTie(engine: BrainEngine, bucket: { source_id: string; entity_slug: string }, cluster: FactRow[], tie: SameInstantTie): Promise<void> {
await engine.transaction(tx => holdConsolidationTie(tx, bucket.source_id, bucket.entity_slug, cluster.map(f => f.id),
{ entity_slug: bucket.entity_slug, tied: tie.ids, valid_from: new Date(tie.at).toISOString(), metric: tie.metric }));
}
function heldCluster(bucket: { source_id: string; entity_slug: string }, tie: SameInstantTie): SameInstantCluster {
const validFrom = new Date(tie.at).toISOString();
const claim = [tie.metric, tie.unit, tie.period].filter(Boolean).join(', ');
return { source_id: bucket.source_id, entity_slug: bucket.entity_slug, metric: tie.metric, unit: tie.unit, period: tie.period, valid_from: validFrom,
fact_ids: tie.ids, fix: { ask_user: true, argv: ['gbrain', 'forget', '<FACT_ID>'], consent: [], actor: 'agent', requires_exclusive: false,
inputs: [{ name: 'FACT_ID', how: `One of facts ${tie.ids.join(', ')}: the statement the user says is no longer current; for exact duplicates, any one of them.` }],
why: 'Consolidation cannot order statements of one metric at one effective time, so it left the cluster unchanged; forgetting the stale one lets the next consolidate run promote the rest.',
user_message: `Facts ${tie.ids.join(', ')} on ${bucket.entity_slug} state ${claim} for the same effective time (${validFrom}), and gbrain cannot tell which one is current, so it left them unconsolidated. Which one should stay? The other is forgotten, and the next consolidate run promotes the cluster.` } };
}
/** The typed-claim columns of a fact (FactRow does not carry them). */
export interface ClaimShape {
metric: string | null;
unit: string | null;
period: string | null;
}
async function loadClaimShapes(engine: BrainEngine, ids: number[]): Promise<Map<number, ClaimShape>> {
const rows = ids.length === 0 ? [] : await engine.executeRaw<{
id: number; claim_metric: string | null; claim_unit: string | null; claim_period: string | null;
}>(`SELECT id, claim_metric, claim_unit, claim_period FROM facts WHERE id = ANY($1::bigint[])`, [ids]);
return new Map(rows.map(r => [Number(r.id), { metric: r.claim_metric, unit: r.claim_unit, period: r.claim_period }]));
}
/**
* #5363: two facts may share a take only when their typed claims agree.
* Untyped facts (no metric) cluster with untyped facts; a typed fact
* clusters only with facts of the same metric, unit and period (a monthly
* and an annual MRR, or USD and EUR, never merge however close their text
* embeddings are). A fact whose claim columns are unknown is treated as
* untyped.
*/
export function claimsCompatible(a: ClaimShape | undefined, b: ClaimShape | undefined): boolean {
const metricA = a?.metric ?? null;
const metricB = b?.metric ?? null;
if (metricA === null || metricB === null) return metricA === metricB;
return metricA === metricB && (a?.unit ?? null) === (b?.unit ?? null) && (a?.period ?? null) === (b?.period ?? null);
}
/**
* #5363: the knob was unreachable (cycle.ts never passed it); the phase reads
* its own config key. A malformed stored value keeps the default and is
* reported in details (`cluster_threshold_invalid`) instead of failing the cycle.
*/
async function resolveClusterThreshold(
engine: BrainEngine,
override: number | undefined,
): Promise<{ threshold: number; thresholdInvalid?: string }> {
if (override !== undefined) return { threshold: override };
const configured = await engine.getConfig(CLUSTER_THRESHOLD_KEY);
if (configured == null || configured.trim() === '') return { threshold: DEFAULT_CLUSTER_THRESHOLD };
try { return { threshold: parseClusterThreshold(configured) }; }
catch { return { threshold: DEFAULT_CLUSTER_THRESHOLD, thresholdInvalid: configured }; }
}
/** One cluster a dry run would promote (`details.clusters`, capped at DRY_RUN_CLUSTER_CAP). */
export interface ConsolidateClusterPreview {
source_id: string;
entity_slug: string;
/** The would-be take claim; null when the fact is private and the caller is not the local CLI. */
claim: string | null;
members: Array<{ id: number; visibility: string; fact: string | null }>;
}
export const DRY_RUN_CLUSTER_CAP = 50;
/**
* #6023: untyped facts have no metric/unit/period guard, so two event facts
* that differ only in an identifier, date or amount could cluster on a 0.85
* embedding and the writeback would expire the older one. They cluster only
* at the explicit-duplicate bar (never below the configured threshold).
*/
export function untypedClusterThreshold(threshold: number): number {
return Math.max(threshold, EXPLICIT_DUPLICATE_THRESHOLD);
}
/**
* Greedy cosine clustering. Iterate facts sorted by valid_from DESC; each
* fact joins the first cluster whose centroid (the first member, for
* simplicity) is within `threshold` cosine and whose typed claim is
* compatible (#5363). Otherwise starts a new cluster.
*
* #6023: an untyped pair additionally needs cosine ≥ the explicit-duplicate
* bar (`untypedClusterThreshold`) and must not diverge in a negation, number,
* date or amount token (`claimsDiverge`): "Invoice A due Monday" and
* "Invoice B due Tuesday" stay two facts. Typed facts keep the configured
* threshold and no token guard, because a metric series is expected to
* differ in its values (that is the trajectory the writeback records).
*
* Facts with no embedding cluster on their own (single-element cluster);
* the consolidate phase only writes takes from clusters of size ≥ 2, so
* no-embedding singletons sit out the cycle. v0.32+ fact-extraction
* pipeline ensures embeddings are computed at insertFact time.
*/
export function clusterFacts(facts: FactRow[], threshold: number, claims: Map<number, ClaimShape> = new Map()): FactRow[][] {
const sorted = [...facts].sort((a, b) => b.valid_from.getTime() - a.valid_from.getTime());
const clusters: FactRow[][] = [];
for (const f of sorted) {
if (!f.embedding || !f.embedding_model || f.embedded_text_hash !== createHash('md5').update(f.fact).digest('hex')) {
clusters.push([f]);
continue;
}
let placed = false;
for (const c of clusters) {
const head = c[0];
if (!head.embedding || f.embedding_model !== head.embedding_model
|| head.embedded_text_hash !== createHash('md5').update(head.fact).digest('hex')
|| f.embedding.length !== head.embedding.length
|| !claimsCompatible(claims.get(f.id), claims.get(head.id))) continue;
const untyped = (claims.get(f.id)?.metric ?? null) === null;
if (untyped && claimsDiverge(f.fact, head.fact)) continue;
if (cosineSimilarity(f.embedding, head.embedding) >= (untyped ? untypedClusterThreshold(threshold) : threshold)) {
c.push(f);
placed = true;
break;
}
}
if (!placed) clusters.push([f]);
}
return clusters;
}
function clamp01(x: number): number {
if (!Number.isFinite(x)) return 0.5;
if (x < 0) return 0;
if (x > 1) return 1;
return x;
}