Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
22.5 KB
Raw Blame
/**
 * #5888 — capture-lane duplicate policy, shared by every write path that
 * stores automatically captured facts (`hook:writeback`, `sweep:corpus`,
 * `hook:compact`) and by the hot-memory read that injects them.
 *
 * Write side (`applyCaptureDedup`, called by the backstop before its
 * managed/unmanaged branch): an extracted candidate is DROPPED only when an
 * active fact in the same source has the same `gbrain_fact_fingerprint` and
 *   - sits on the same entity (or both have none), or on a different entity
 *     whose page title or alias the claim names (subject-relative claims such
 *     as "Prefers email" captured for two people are never merged);
 *   - is visible where the candidate is (a `private` candidate compares with
 *     `world` and `private` facts, a `world` candidate only with `world`);
 *   - was written within 15 minutes of the source turn, or by a capture lane
 *     in the same conversation (no time bound inside one conversation).
 * Near duplicates (cosine >= 0.92 on the same entity, claims not differing
 * in a negation, number or date token) are never dropped: they are counted
 * in shadow mode (`near_duplicate` on the `writeback_dedup` heartbeat event)
 * so a threshold can be measured before it ever deletes anything. A failed
 * dedup read inserts the candidate (fail open) and warns with the lane.
 * Explicit lanes (`remember`, extract_facts, sync) never pass through here.
 * #7043: conversation-fact extraction reuses the exact-fingerprint rule with
 * no time bound (`anchor: null`), skipping a candidate an active fact of the
 * source already states (conversation-publication.ts).
 *
 * Read side (`collapseHotFacts`): rows with the same fingerprint on the same
 * entity, or on different entities when the claim names one of them,
 * collapse to the newest row, which carries every entity slug of its group.
 * `collapseFactReads` applies it to reads that span sources and
 * visibilities (recall, search's saved facts, #7043).
 */

import type { BrainEngine, FactAttribution, FactRow, FactVisibility } from '../engine.ts';
import { corpusFileSessionId } from '../context/corpus-segments.ts';
import { writeHeartbeat } from '../context/hook-heartbeat.ts';
import { normalizeAlias } from '../search/alias-normalize.ts';
import { cosineSimilarity } from './classify.ts';
import { onOwnedMemoryPageSql } from '../access/member-facts.ts';

export const CAPTURE_LANES = ['hook:writeback', 'sweep:corpus', 'hook:compact'] as const;
export const CAPTURE_DEDUP_WINDOW_MS = 15 * 60 * 1000;
/** Shadow-mode near-duplicate threshold (measured, never a drop). */
export const NEAR_DUPLICATE_THRESHOLD = 0.92;
/** The bar consolidate clusters untyped facts at (#6023). Explicit-lane supersession reads the per-model threshold instead (supersession-threshold.ts). */
export const EXPLICIT_DUPLICATE_THRESHOLD = 0.95;
const CAPTURE_DEDUP_SCAN_LIMIT = 50;

export function isCaptureLane(source: string | null | undefined): boolean {
  return (CAPTURE_LANES as readonly string[]).includes(source ?? '');
}

/**
 * One key per harness conversation across the capture lanes: the corpus sweep
 * tags rows `sweep:corpus:<file>`, where the file is `<session>.txt`, a
 * checkpoint segment or a writeback turn file of that session.
 */
export function canonicalConversationKey(sessionId: string | null | undefined): string | null {
  if (!sessionId) return null;
  const bare = sessionId.startsWith('sweep:corpus:') ? sessionId.slice('sweep:corpus:'.length) : sessionId;
  const key = bare.endsWith('.txt') ? corpusFileSessionId(bare) : bare;
  return key || null;
}

const NEGATION_TOKENS = /\b(?:no longer|not|no|never|stopped|isn't|doesn't|won't|isnt|doesnt|wont)\b/g;
const DATE_WORDS = /\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|(?:mon|tues|wednes|thurs|fri|satur|sun)days?|today|tomorrow|yesterday)\b/g;
const NUMBER_TOKENS = /[$€£¥]?\d[\d,.:/-]*\s?(?:k|m|bn|b|%|percent)?\b/g;
const NUMBER_WORDS = /\b(?:zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|twenty|thirty|forty|fifty|hundred|thousand|million|billion|dozen|half|twice|first|second|third|last)\b/g;

function guardTokens(text: string): string {
  const t = text.normalize('NFKC').toLowerCase().replace(/[’‘]/g, "'");
  const pick = (re: RegExp) => (t.match(re) ?? []).map(s => s.replace(/\s+/g, '').replace(/[.,]+$/, ''));
  return JSON.stringify([pick(NEGATION_TOKENS).sort(), [...pick(NUMBER_TOKENS), ...pick(NUMBER_WORDS), ...pick(DATE_WORDS).map(d => d.replace(/days$/, 'day'))].sort()]);
}

/** V1 guard: true when two claims differ in a negation, number, date or amount token. */
export function claimsDiverge(a: string, b: string): boolean {
  return guardTokens(a) !== guardTokens(b);
}

export type CosineVerdict = 'duplicate' | 'near_duplicate' | 'distinct';

/**
 * The one cosine policy for every fact-write decision point. Capture lanes
 * never return `duplicate`. Explicit lanes return `duplicate` at or above
 * `explicitThreshold`, the embedding model's calibrated supersession
 * threshold (supersession-threshold.ts); null (an uncalibrated model) never
 * returns `duplicate`.
 */
export function cosineVerdict(source: string | null | undefined, score: number, claim: string, existing: string, explicitThreshold: number | null): CosineVerdict {
  if (!isCaptureLane(source)) return explicitThreshold !== null && score >= explicitThreshold ? 'duplicate' : 'distinct';
  return score >= NEAR_DUPLICATE_THRESHOLD && !claimsDiverge(claim, existing) ? 'near_duplicate' : 'distinct';
}

/** #6333 shadow floor: same-page rewordings sit at cosine 0.86–0.93 (lane C), under every calibrated dedup threshold. */
export const SAME_PAGE_REWORDING_THRESHOLD = 0.85;

/**
 * #6333 [UC2] shadow predicate, never a drop: would the deterministic rule
 * "same source-qualified provenance (lane + session + visibility), cosine at
 * or above SAME_PAGE_REWORDING_THRESHOLD, no diverging guard token" have
 * treated `claim` as a duplicate of `existing`? The backstop counts it as
 * `would_dedup_same_page` on explicit lanes only. Measured false positives
 * (Lisbon→Porto, approved→rejected, email→phone, all invisible to
 * claimsDiverge) are why the rule measures and does not act.
 */
export function wouldDedupSamePage(
  ctx: { source: string | null | undefined; sessionId: string | null | undefined; visibility: FactVisibility },
  existing: { source: string; source_session: string | null; visibility: FactVisibility; fact: string },
  score: number,
  claim: string,
): boolean {
  if (!ctx.source || !ctx.sessionId || isCaptureLane(ctx.source)) return false;
  if (existing.source !== ctx.source || existing.source_session !== ctx.sessionId || existing.visibility !== ctx.visibility) return false;
  return score >= SAME_PAGE_REWORDING_THRESHOLD && !claimsDiverge(claim, existing.fact);
}

function nameKey(text: string): string {
  return ` ${normalizeAlias(text).replace(/[^\p{L}\p{N}]+/gu, ' ').trim()} `;
}

/** True when the claim contains one of the entity's names as whole words. */
export function claimNamesEntity(claim: string, names: readonly string[]): boolean {
  const haystack = nameKey(claim);
  return names.some(name => {
    const needle = nameKey(name);
    return needle.trim().length > 0 && haystack.includes(needle);
  });
}

/** Title + aliases of each live entity page, keyed by slug. */
export async function entityNames(engine: BrainEngine, sourceId: string, slugs: string[]): Promise<Map<string, string[]>> {
  const names = new Map<string, string[]>();
  const wanted = [...new Set(slugs)];
  if (!wanted.length) return names;
  const add = (slug: string, name: string | null) => {
    if (name) names.set(slug, [...(names.get(slug) ?? []), name]);
  };
  const pages = await engine.executeRaw<{ slug: string; title: string | null }>(
    'SELECT slug,title FROM pages WHERE source_id=$1 AND slug=ANY($2::text[]) AND deleted_at IS NULL', [sourceId, wanted]);
  for (const page of pages) add(page.slug, page.title);
  try {
    const aliases = await engine.executeRaw<{ slug: string; alias_norm: string }>(
      'SELECT slug,alias_norm FROM page_aliases WHERE source_id=$1 AND slug=ANY($2::text[])', [sourceId, pages.map(p => p.slug)]);
    for (const alias of aliases) add(alias.slug, alias.alias_norm);
  } catch { /* pre-v110 brain: titles only */ }
  return names;
}

export interface CaptureCandidate {
  fact: string;
  entitySlug: string | null;
  visibility: 'private' | 'world';
  embedding?: Float32Array | null;
  embeddingModel?: string | null;
  entityInferred?: unknown;
  /** Speaker of the candidate; a fact another known speaker asserted never matches. */
  attributedTo?: FactAttribution | null;
}
export interface CaptureScope {
  engine: BrainEngine;
  sourceId: string;
  lane: string;
  sessionId: string | null;
  /** The source turn's own time when the lane knows it, else check time; null drops the time bound (#7043). */
  anchor: Date | null;
  /** Ignore this page's own conversation-extractor rows: a replacement is about to clear them (#7043). */
  replacingPage?: string;
}
export interface CaptureMatch { id: number; rule: 'same_entity' | 'named_entity'; }

/** The exact-fingerprint drop rule (one bounded indexed read). */
export async function findCaptureDuplicate(scope: CaptureScope, candidate: CaptureCandidate): Promise<CaptureMatch | null> {
  const visibilities = candidate.visibility === 'world' ? ['world'] : ['world', 'private'];
  const rows = await scope.engine.executeRaw<{ id: number; entity_slug: string | null; source: string; source_session: string | null; created_at: Date | string }>(
    `SELECT id,entity_slug,source,source_session,created_at FROM facts WHERE source_id=$1 AND visibility=ANY($2::text[])
      AND gbrain_fact_fingerprint(fact)=gbrain_fact_fingerprint($3) AND expired_at IS NULL AND (valid_until IS NULL OR valid_until>now())
      AND ($4::text IS NULL OR attributed_to IS NULL OR attributed_to=$4)
      AND ($5::text IS NULL OR source_markdown_slug IS DISTINCT FROM $5 OR source NOT LIKE 'cli:extract-conversation-facts%')
      AND NOT ${onOwnedMemoryPageSql('facts')}
      ORDER BY created_at DESC, id DESC LIMIT ${CAPTURE_DEDUP_SCAN_LIMIT}`,
    [scope.sourceId, visibilities, candidate.fact, candidate.attributedTo ?? null, scope.replacingPage ?? null]);
  const conversation = canonicalConversationKey(scope.sessionId);
  const anchor = scope.anchor;
  const recent = anchor === null ? rows : rows.filter(row => Math.abs(new Date(row.created_at).getTime() - anchor.getTime()) <= CAPTURE_DEDUP_WINDOW_MS
    || conversation !== null && isCaptureLane(row.source) && canonicalConversationKey(row.source_session) === conversation);
  const same = recent.find(row => (row.entity_slug ?? null) === candidate.entitySlug);
  if (same) return { id: Number(same.id), rule: 'same_entity' };
  const others = recent.filter(row => row.entity_slug);
  if (!others.length) return null;
  const names = await entityNames(scope.engine, scope.sourceId, others.map(row => row.entity_slug!));
  const named = others.find(row => claimNamesEntity(candidate.fact, names.get(row.entity_slug!) ?? []));
  return named ? { id: Number(named.id), rule: 'named_entity' } : null;
}

/** Shadow mode: the id of a same-entity near duplicate, never a drop. */
export async function findNearDuplicate(scope: CaptureScope, candidate: CaptureCandidate): Promise<number | null> {
  if (!candidate.entitySlug || !candidate.embedding || candidate.entityInferred) return null;
  const found = await scope.engine.findCandidateDuplicates(scope.sourceId, candidate.entitySlug, candidate.fact,
    { embedding: candidate.embedding, embeddingModel: candidate.embeddingModel, k: 5, attributedTo: candidate.attributedTo ?? null });
  let best: { id: number; score: number; fact: string } | null = null;
  for (const row of found) {
    if (!row.embedding || row.expired_at || candidate.visibility === 'world' && row.visibility !== 'world') continue;
    const score = cosineSimilarity(candidate.embedding, row.embedding);
    if (!best || score > best.score) best = { id: row.id, score, fact: row.fact };
  }
  return best && cosineVerdict(scope.lane, best.score, candidate.fact, best.fact, null) === 'near_duplicate' ? best.id : null;
}

export interface CaptureDedupResult<T> { kept: T[]; duplicateIds: number[]; nearDuplicates: number; }

/**
 * The shared pre-branch check. `resolve` maps an extracted entity reference
 * to the slug the writers will use (null when unresolved).
 */
export async function applyCaptureDedup<T extends { fact: string; entity_slug?: string | null; embedding?: Float32Array | null; embedding_model?: string | null; entity_inferred?: unknown; attributed_to?: FactAttribution | null }>(
  scope: CaptureScope, facts: T[], visibility: 'private' | 'world', resolve: (entity: string | null | undefined) => Promise<string | null>,
): Promise<CaptureDedupResult<T>> {
  const started = Date.now();
  const result: CaptureDedupResult<T> = { kept: [], duplicateIds: [], nearDuplicates: 0 };
  let failed = false;
  for (const fact of facts) {
    const candidate: CaptureCandidate = { fact: fact.fact, entitySlug: await resolve(fact.entity_slug), visibility,
      embedding: fact.embedding, embeddingModel: fact.embedding_model, entityInferred: fact.entity_inferred, attributedTo: fact.attributed_to ?? null };
    try {
      const match = await findCaptureDuplicate(scope, candidate);
      if (match) {
        result.duplicateIds.push(match.id);
        console.warn(`[facts] capture dedup: lane=${scope.lane} dropped a duplicate of fact #${match.id} (rule=${match.rule})`);
        continue;
      }
      if (await findNearDuplicate(scope, candidate) !== null) result.nearDuplicates++;
    } catch (err) {
      failed = true;
      console.warn(`[facts] capture dedup read failed: lane=${scope.lane} inserting the candidate (${err instanceof Error ? err.message : String(err)})`);
    }
    result.kept.push(fact);
  }
  if (result.duplicateIds.length || result.nearDuplicates || failed) {
    await writeHeartbeat({ ts: new Date().toISOString(), event: 'writeback_dedup', outcome: failed ? 'degraded' : 'ok',
      reason: failed ? `${scope.lane}:read_failed` : scope.lane, duration_ms: Date.now() - started,
      duplicate: result.duplicateIds.length, near_duplicate: result.nearDuplicates }, { trim: false });
  }
  return result;
}

/**
 * The backstop's entry point: capture lanes only, slugs resolved the way both
 * writers resolve them, anchored on the turn time when the lane knows it.
 */
export async function dedupCapturedFacts<T extends { fact: string; entity_slug?: string | null; embedding?: Float32Array | null; embedding_model?: string | null; entity_inferred?: unknown; attributed_to?: FactAttribution | null }>(
  ctx: { engine: BrainEngine; sourceId: string; source: string; sessionId: string | null; turnAt?: Date },
  facts: T[], visibility: 'private' | 'world',
  resolveEntity: (engine: BrainEngine, sourceId: string, raw: string) => Promise<{ slug: string; source: string } | null>,
): Promise<{ facts: T[]; dropped: number[] }> {
  if (!isCaptureLane(ctx.source)) return { facts, dropped: [] };
  const result = await applyCaptureDedup({ engine: ctx.engine, sourceId: ctx.sourceId, lane: ctx.source, sessionId: ctx.sessionId,
    anchor: ctx.turnAt ?? new Date() }, facts, visibility, async entity => {
    const resolved = entity ? await resolveEntity(ctx.engine, ctx.sourceId, entity) : null;
    return resolved && resolved.source !== 'fallback_slugify' ? resolved.slug : null;
  });
  return { facts: result.kept, dropped: result.duplicateIds };
}

/** Folds the pre-branch drops into a writer's counts (null: every candidate was dropped). */
export function withCaptureDrops<R extends { inserted: number; duplicate: number; superseded: number; fact_ids: number[]; entity_slugs: string[] }>(
  dropped: number[], published: R | null,
): R | { inserted: number; duplicate: number; superseded: number; fact_ids: number[]; entity_slugs: string[] } {
  if (!published) return { inserted: 0, duplicate: dropped.length, superseded: 0, fact_ids: dropped, entity_slugs: [] };
  return { ...published, duplicate: published.duplicate + dropped.length, fact_ids: [...dropped, ...published.fact_ids] };
}

export type HotFactRow = FactRow & { fact_fingerprint?: string | null };
/** What the collapse reads of a row. */
export type CollapsibleFact = Pick<FactRow, 'id' | 'fact' | 'entity_slug' | 'attributed_to' | 'created_at'> & { fact_fingerprint?: string | null };
export type CollapsedHotFact<R extends CollapsibleFact> = R & { entity_slugs?: string[] };

/**
 * V2 hot-memory collapse: one representative (the newest row) per
 * (fingerprint, entity) group; groups of different entities merge only when
 * the claim names one of them. Input order is preserved by representative.
 * A group holding claims from two different known speakers (user and
 * assistant) also splits by speaker, so one never hides the other.
 */
/** `members` (optional) receives raw row id -> emitted representative id, so delta can advance its facts keyset through raw rows. */
export async function collapseHotFacts<R extends CollapsibleFact>(engine: BrainEngine, sourceId: string, rows: R[], members?: Map<number, number>): Promise<CollapsedHotFact<R>[]> {
  const byFingerprint = new Map<string, R[]>();
  for (const row of rows) {
    const key = row.fact_fingerprint ?? `id:${row.id}`;
    byFingerprint.set(key, [...(byFingerprint.get(key) ?? []), row]);
  }
  const multiEntity = [...byFingerprint.values()].filter(group => new Set(group.map(r => r.entity_slug)).size > 1);
  const names = multiEntity.length
    ? await entityNames(engine, sourceId, multiEntity.flatMap(group => group.flatMap(r => r.entity_slug ? [r.entity_slug] : []))).catch(() => new Map<string, string[]>())
    : new Map<string, string[]>();
  const clusterOf = new Map<R, R[]>();
  for (const group of byFingerprint.values()) {
    const named = group.some(r => r.entity_slug && claimNamesEntity(r.fact, names.get(r.entity_slug) ?? []));
    const speakers = new Set(group.flatMap(r => r.attributed_to ? [r.attributed_to] : []));
    const clusters = new Map<string, R[]>();
    for (const row of group) {
      const entityKey = named ? '*' : row.entity_slug ?? '';
      const key = speakers.size > 1 ? `${entityKey}\u0000${row.attributed_to ?? ''}` : entityKey;
      clusters.set(key, [...(clusters.get(key) ?? []), row]);
    }
    for (const cluster of clusters.values()) for (const row of cluster) clusterOf.set(row, cluster);
  }
  const out: CollapsedHotFact<R>[] = [];
  const emitted = new Set<R[]>();
  for (const row of rows) {
    const cluster = clusterOf.get(row)!;
    if (emitted.has(cluster)) continue;
    emitted.add(cluster);
    const newest = cluster.reduce((a, b) => b.created_at.getTime() > a.created_at.getTime() || b.created_at.getTime() === a.created_at.getTime() && b.id > a.id ? b : a);
    if (members) for (const r of cluster) members.set(r.id, newest.id);
    const slugs = [...new Set([newest.entity_slug, ...cluster.map(r => r.entity_slug)].filter((s): s is string => !!s))];
    out.push(slugs.length > 1 ? { ...newest, entity_slugs: slugs } : newest);
  }
  return out;
}

/** A row of a read that spans sources and visibilities; `created_at` may arrive as text from a raw read. */
export type FactReadRow = Omit<CollapsibleFact, 'created_at'> & { source_id: string; visibility: string; created_at: Date | string;
  expired_at?: Date | string | null; valid_until?: Date | string | null; entity_slugs?: string[]; collapsed_count?: number; collapsed_ids?: number[] };
/** `collapsed_ids`: every row id the representative stands for (internal; callers drop it from responses). */
export type CollapsedFactRead<R> = R & { entity_slugs?: string[]; collapsed_count?: number; collapsed_ids?: number[] };

/**
 * #7043: `collapseHotFacts` for recall and search. Active rows collapse only
 * within one (source, visibility), so a remote caller's already-filtered rows
 * never meet a hidden copy; expired or lapsed rows pass through unchanged.
 * Rows without `fact_fingerprint` get it from one read of their text. Order
 * is preserved by representative (the group's newest row, holding its
 * first member's place); a representative standing for several rows carries
 * `collapsed_count` and `collapsed_ids`, and `entity_slugs` when they name
 * more than one entity. A row that is already a representative counts as the
 * rows it stands for, so collapsing a merged list of two collapsed reads
 * counts each row once. A failed fingerprint read returns the rows unchanged.
 */
export async function collapseFactReads<R extends FactReadRow>(engine: BrainEngine, rows: R[]): Promise<CollapsedFactRead<R>[]> {
  const now = Date.now();
  const active = rows.filter(r => !r.expired_at && (!r.valid_until || new Date(r.valid_until).getTime() > now));
  if (active.length < 2) return rows;
  const missing = active.filter(r => !r.fact_fingerprint);
  let computed: string[] = [];
  try {
    if (missing.length) computed = (await engine.executeRaw<{ fp: string }>(
      'SELECT gbrain_fact_fingerprint(c) AS fp FROM unnest($1::text[]) WITH ORDINALITY AS t(c, n) ORDER BY n', [missing.map(r => r.fact)])).map(r => r.fp);
  } catch {
    return rows;
  }
  const fingerprints = new Map(missing.map((r, i) => [r.id, computed[i]]));
  const partitions = new Map<string, CollapsibleFact[]>();
  for (const r of active) {
    const key = `${r.source_id}\u0000${r.visibility}`;
    partitions.set(key, [...(partitions.get(key) ?? []), { id: r.id, fact: r.fact, entity_slug: r.entity_slug, attributed_to: r.attributed_to,
      created_at: new Date(r.created_at), fact_fingerprint: r.fact_fingerprint ?? fingerprints.get(r.id) ?? null }]);
  }
  const members = new Map<number, number>();
  for (const [key, part] of partitions) await collapseHotFacts(engine, key.slice(0, key.indexOf('\u0000')), part, members);
  const byId = new Map(rows.map(r => [r.id, r]));
  const groups = new Map<number, R[]>();
  for (const [id, rep] of members) groups.set(rep, [...(groups.get(rep) ?? []), byId.get(id)!]);
  const out: CollapsedFactRead<R>[] = [];
  const emitted = new Set<number>();
  for (const row of rows) {
    const rep = members.get(row.id);
    if (rep === undefined) { out.push(row); continue; }
    if (emitted.has(rep)) continue;
    emitted.add(rep);
    const group = groups.get(rep)!;
    const kept = byId.get(rep)!;
    const slugs = [...new Set([kept.entity_slug, ...group.flatMap(r => r.entity_slugs ?? [r.entity_slug])].filter((s): s is string => !!s))];
    const ids = [...new Set(group.flatMap(r => r.collapsed_ids ?? [r.id]))].sort((a, b) => a - b);
    out.push({ ...kept, ...(slugs.length > 1 ? { entity_slugs: slugs } : {}), ...(ids.length > 1 ? { collapsed_count: ids.length, collapsed_ids: ids } : {}) });
  }
  return out;
}