Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.6 KB
Raw Blame
import type { BrainEngine, FactAttribution, FactRow } from '../engine.ts';
import { verbError, type OperationError } from '../ops/contract.ts';
import { isAvailable, embedOne, getEmbeddingModel } from '../ai/gateway.ts';
import { cosineSimilarity } from './classify.ts';
import { FACT_WITHDRAWN_MESSAGE, FACT_WITHDRAWN_SUGGESTION, isFactWithdrawn } from './withdrawal.ts';
import { assertFactNotPurged } from './withdrawal.ts';
import { cosineVerdict } from './capture-dedup.ts';
import { interleaveFusion } from '../search/fusion-lists.ts';
import { readSupersessionThreshold } from './supersession-threshold.ts';
import { ownerReadPolicyFragment, readPolicyFragment, remoteProvenanceFloorSql } from '../search/private-visibility.ts';
import { folderAccessForAll } from '../access/caller.ts';
import { memberIdentity } from '../access/members.ts';
import { ownerMemberId } from '../access/owner.ts';
import { MEMBER_PAGE_PREFIX } from '../access/principals.ts';
import { pageNotVisibleError } from '../persistence/page-visibility.ts';
import { slugUnderBoundPrefixes } from '../ops/context.ts';
import { ownedPageWithdrawalSubject, onOwnedMemoryPageSql, withdrawalCoversFactSql, withdrawalReachedFactSql } from '../access/member-facts.ts';
import { isHoldingSlug } from '../facts-fence.ts';
import { sameClaimSlot } from './typed-claim.ts';

export type FactCandidate = FactRow & { source_markdown_slug: string | null; row_num: number | null };
export interface FactDecision { status: 'inserted' | 'duplicate' | 'superseded'; candidate: FactCandidate | null; }
export interface SingleFactIntent {
  fact: string; kind: FactRow['kind']; visibility: FactRow['visibility']; entity_slug: string | null;
  /** Speaker of the new claim; a fact another known speaker asserted is never its duplicate. */
  attributed_to?: FactAttribution | null;
  /**
   * GST-359 C2: the member page a member's fact goes to (access/member-facts.ts). Its duplicates and supersession
   * candidates are only facts on that page (the writer's own, so the readers filter is not applied to them); a write
   * with no page_slug never matches a fact on any member page.
   */
  page_slug?: string;
  /** A typed new fact's claim slot (remember): it never supersedes a typed fact in another slot (typed-claim.ts sameClaimSlot). Unset (untyped): not checked. */
  slot?: { claim_metric: string | null; claim_unit: string | null; claim_period: string | null; valid_from: Date | string };
}
/** Preparation time kept back after the embed (a quarter of what is left, at most this): the dedup reads still run. */
const EMBED_RESERVE_MS = 5_000;
/**
 * Provider work belongs to preparation, never to a page/source transaction. `disabled`: the brain opted out of embedding.
 * `deadlineAt` (#8135): the preparation budget's end. The embed gets its own timeout short of it, and that timeout takes
 * the no-vector fallback like any embed failure; without it a slow embedder let the budget's abort win, the attempt was
 * charged, and the write ended `preparation_stalled` with nothing saved. The caller's own abort still propagates.
 */
export async function prepareFactEmbedding(fact: string, signal?: AbortSignal, disabled = false, deadlineAt?: number): Promise<{ embedding: Float32Array | null; embedding_model: string | null; degraded: boolean }> {
  signal?.throwIfAborted();
  const remaining = deadlineAt === undefined ? Infinity : deadlineAt - Date.now();
  const embedMs = remaining - Math.min(EMBED_RESERVE_MS, Math.floor(remaining / 4));
  if (!disabled && isAvailable('embedding') && embedMs > 0) {
    try {
      const model = getEmbeddingModel();
      const abortSignal = Number.isFinite(embedMs) ? AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(embedMs)]) : signal;
      return { embedding: await embedOne(fact, { abortSignal, embeddingModel: model, inputType: 'document' }), embedding_model: model, degraded: false };
    } catch { signal?.throwIfAborted(); }
  }
  return { embedding: null, embedding_model: null, degraded: true };
}
/**
 * `excludePrivate`: the caller cannot read private pages. A withdrawal whose withdrawn rows all sit on pages it
 * cannot read is refused exactly as a page it may not see is, so the refusal never confirms a claim it cannot see.
 * `remote`: GST-2346, a remote save onto a member page meets only memberPageWithdrawal's withdrawals.
 */
export async function assertFactNotWithdrawn(engine: BrainEngine, sourceId: string, input: SingleFactIntent, opts: { excludePrivate?: boolean; remote?: boolean } = {}): Promise<void> {
  await assertFactNotPurged(engine, sourceId, input);
  if (opts.remote && input.page_slug?.startsWith(MEMBER_PAGE_PREFIX)) {
    const met = await memberPageWithdrawal(engine, sourceId, input, input.page_slug);
    if (met) throw withdrawnRefusal(met);
    return;
  }
  if (await isFactWithdrawn(engine, sourceId, input.visibility, input.fact, input.entity_slug, ownedPageWithdrawalSubject(input.page_slug))) throw await factWithdrawnRefusal(engine, sourceId, input, opts);
}

type WithdrawalRow = { subject: string; withdrawn_at: string; visibility: 'world' | 'private' };

function withdrawnRefusal(row: WithdrawalRow | undefined): OperationError {
  const error = verbError('invalid_params', FACT_WITHDRAWN_MESSAGE, FACT_WITHDRAWN_SUGGESTION);
  if (row) error.withdrawn = { at: new Date(row.withdrawn_at).toISOString(), subject: row.subject === '*' ? 'all_entities' : row.subject, visibility: row.visibility };
  return error;
}

/**
 * GST-2346: the withdrawal a remote save onto member page `page` meets, or null. What the member gets back never depends
 * on a claim it could not read, so the save meets the page's own withdrawals and a withdrawal of another subject only
 * when a row it withdrew is one the page's member reads: a row on that page, or a row on pages it reads (a shared one,
 * or for the enabled brain owner any one, under GST-2490's owner read rule). Otherwise the save is the member's own
 * row, which the insert guards leave active (facts/withdrawal-schema.ts). Judged by the page's member, live: a member
 * disabled or removed reads nothing.
 */
async function memberPageWithdrawal(engine: BrainEngine, sourceId: string, input: SingleFactIntent, page: string): Promise<WithdrawalRow | null> {
  const memberId = page.slice(MEMBER_PAGE_PREFIX.length);
  const identity = (await memberIdentity(engine, memberId).catch(() => null))?.identity ?? [];
  const satisfied = identity.length ? (await folderAccessForAll(engine, [identity])).satisfiedRuleIds : [];
  const owner = identity.length > 0 && await ownerMemberId(engine) === memberId;
  const reads = (alias: string) => owner ? ownerReadPolicyFragment(alias, identity, satisfied) : readPolicyFragment(alias, identity, true, satisfied);
  const readable = `(f.source_markdown_slug = $5::text OR (${owner ? 'TRUE' : `f.visibility = 'world'`}
    AND NOT EXISTS (SELECT 1 FROM pages pp WHERE pp.source_id = f.source_id AND pp.slug = f.source_markdown_slug AND NOT ${reads('pp')})
    AND NOT EXISTS (SELECT 1 FROM pages ep WHERE ep.source_id = f.source_id AND ep.slug = f.entity_slug AND NOT ${reads('ep')})))`;
  const [row] = await engine.executeRaw<WithdrawalRow>(`SELECT w.subject,w.visibility,w.withdrawn_at::text AS withdrawn_at FROM fact_withdrawals w
    WHERE w.source_id=$1 AND w.visibility=$2 AND w.fact_hash IN (gbrain_fact_fingerprint($3),gbrain_fact_fingerprint_v1($3))
      AND (w.subject=$5::text OR ((w.subject='*' OR w.subject=$4::text) AND EXISTS (SELECT 1 FROM facts f
        WHERE f.source_id=w.source_id AND f.visibility=w.visibility AND f.expired_at IS NOT NULL
          AND gbrain_fact_fingerprint(f.fact)=gbrain_fact_fingerprint($3) AND ${withdrawalReachedFactSql('w', 'f')} AND ${readable})))
    ORDER BY (w.subject<>'*') DESC, w.withdrawn_at DESC LIMIT 1`, [sourceId, input.visibility, input.fact, input.entity_slug, page]);
  return row ?? null;
}

/**
 * The refusal for a claim `isFactWithdrawn` matched. `withdrawn` comes from the withdrawal ledger alone (never a purge):
 * the most specific subject first, then the newest withdrawal, among withdrawals the caller can see the withdrawn rows of
 * (or that no row backs). It covers the exact normalized claim only.
 */
export async function factWithdrawnRefusal(engine: BrainEngine, sourceId: string, input: SingleFactIntent, opts: { excludePrivate?: boolean } = {}): Promise<OperationError> {
  // GST-359 D: a member writing to its own member page sees that page's withdrawn rows.
  const seen = (param: number) => `(${remoteProvenanceFloorSql('f')} OR COALESCE(f.source_markdown_slug = $${param}::text, false))`;
  if (opts.excludePrivate) {
    const [backing] = await engine.executeRaw<{ visible: boolean; hidden: boolean }>(`SELECT
        COALESCE(bool_or(${seen(5)}), false) AS visible,
        COALESCE(bool_or(NOT ${seen(5)}), false) AS hidden
      FROM facts f JOIN fact_withdrawals w ON w.source_id=f.source_id AND w.visibility=f.visibility
        AND w.fact_hash IN (gbrain_fact_fingerprint(f.fact),gbrain_fact_fingerprint_v1(f.fact))
        AND ${withdrawalReachedFactSql('w', 'f')}
      WHERE f.source_id=$1 AND f.visibility=$2 AND f.expired_at IS NOT NULL
        AND gbrain_fact_fingerprint(f.fact)=gbrain_fact_fingerprint($3) AND (w.subject='*' OR w.subject=$4::text OR w.subject=$5::text)`,
    [sourceId, input.visibility, input.fact, input.entity_slug, ownedPageWithdrawalSubject(input.page_slug)]);
    if (backing?.hidden && !backing.visible) return pageNotVisibleError();
  }
  const [row] = await engine.executeRaw<WithdrawalRow>(`SELECT w.subject,w.visibility,w.withdrawn_at::text AS withdrawn_at
    FROM fact_withdrawals w LEFT JOIN facts f ON f.source_id=w.source_id AND f.visibility=w.visibility AND f.expired_at IS NOT NULL
      AND w.fact_hash IN (gbrain_fact_fingerprint(f.fact),gbrain_fact_fingerprint_v1(f.fact)) AND ${withdrawalReachedFactSql('w', 'f')}
    WHERE w.source_id=$1 AND w.visibility=$2 AND w.fact_hash IN (gbrain_fact_fingerprint($3),gbrain_fact_fingerprint_v1($3))
      AND (w.subject='*' OR w.subject=$4::text OR w.subject=$6::text)
    GROUP BY w.subject,w.visibility,w.withdrawn_at
    HAVING $5::boolean=false OR COALESCE(bool_or(${seen(6)}), false) OR NOT COALESCE(bool_or(f.id IS NOT NULL), false)
    ORDER BY (w.subject<>'*') DESC, w.withdrawn_at DESC LIMIT 1`,
  [sourceId, input.visibility, input.fact, input.entity_slug, opts.excludePrivate === true, ownedPageWithdrawalSubject(input.page_slug)]);
  return withdrawnRefusal(row);
}
export const CANDIDATE_FUSION_KEY = 'facts.candidate_fusion';
export type CandidateFusion = 'rrf_free' | 'interleave';
/** Candidates the supersession decision scores; the cap bounds the blast radius. */
export const SUPERSESSION_CANDIDATE_K = 5;

/** `facts.candidate_fusion`: `rrf_free` (default) is the single cosine arm; unknown values read as the default. */
export async function readCandidateFusion(engine: BrainEngine): Promise<CandidateFusion> {
  try {
    const raw = (await engine.getConfig(CANDIDATE_FUSION_KEY))?.trim();
    return raw === 'interleave' ? 'interleave' : 'rrf_free';
  } catch {
    return 'rrf_free';
  }
}

/**
 * The supersession candidate set for one claim: the cosine arm's top k, or
 * under `interleave` the cosine and keyword arms merged round-robin by fact id
 * and cut to the same k, so the decision still scores at most k rows.
 */
export async function listSupersessionCandidates(engine: BrainEngine, sourceId: string, entitySlug: string, fact: string, embedding: Float32Array, embeddingModel?: string | null, fusion?: CandidateFusion, attributedTo?: FactAttribution | null, scope: { excludeMemberPages?: boolean; onlyPage?: string } = {}): Promise<FactRow[]> {
  const k = SUPERSESSION_CANDIDATE_K;
  const cosine = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, attributedTo, ...scope });
  if ((fusion ?? await readCandidateFusion(engine)) !== 'interleave') return cosine;
  const keyword = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, arm: 'keyword', attributedTo, ...scope });
  return interleaveFusion([cosine, keyword], row => row.id, k);
}

/**
 * SQL-only, so publication can verify the semantic decision under its guard.
 * `lane` is the writer's `facts.source`: capture lanes never drop by cosine (#5888).
 */
export async function decideSingleFact(engine: BrainEngine, sourceId: string, input: SingleFactIntent, embedding: Float32Array | null, embeddingModel?: string | null, lane?: string | null, opts: { excludePrivate?: boolean } = {}): Promise<FactDecision> {
  const [exact] = await engine.executeRaw<FactCandidate>(`SELECT * FROM facts WHERE source_id=$1
    AND entity_slug IS NOT DISTINCT FROM $2 AND visibility=$3 AND expired_at IS NULL
    AND (valid_until IS NULL OR valid_until>now()) AND gbrain_fact_fingerprint(fact)=gbrain_fact_fingerprint($4)
    AND ($5::text IS NULL OR attributed_to IS NULL OR attributed_to=$5)${opts.excludePrivate && !input.page_slug ? ` AND ${remoteProvenanceFloorSql('facts')}` : ''}
    AND ${input.page_slug ? 'source_markdown_slug=$6' : `($6::text IS NULL AND NOT ${onOwnedMemoryPageSql('facts')})`}
    ORDER BY id LIMIT 1`, [sourceId, input.entity_slug, input.visibility, input.fact, input.attributed_to ?? null, input.page_slug ?? null]);
  if (exact) return { status: 'duplicate', candidate: { ...exact, id: Number(exact.id) } };
  if (embedding && input.entity_slug) {
    const candidates = await listSupersessionCandidates(engine, sourceId, input.entity_slug, input.fact, embedding, embeddingModel, undefined, input.attributed_to ?? null,
      input.page_slug ? { onlyPage: input.page_slug } : { excludeMemberPages: true });
    const metadata = await engine.executeRaw<{ id: number; source_markdown_slug: string | null; row_num: number | null; trust_tier: FactRow['trust_tier'];
      claim_metric: string | null; claim_unit: string | null; claim_period: string | null }>(
      `SELECT id,source_markdown_slug,row_num,trust_tier,claim_metric,claim_unit,claim_period FROM facts WHERE source_id=$1 AND id=ANY($2::int[])${opts.excludePrivate && !input.page_slug ? ` AND ${remoteProvenanceFloorSql('facts')}` : ''}`,
      [sourceId, candidates.map(c => c.id)]);
    let candidate: FactCandidate | null = null;
    let score = -1;
    for (const found of candidates) {
      const fence = metadata.find(m => Number(m.id) === found.id);
      if (!fence) continue;
      const c: FactCandidate = { ...found, ...fence, id: found.id };
      // A private candidate must never affect a world's response or expire as a
      // side effect. Cross-page supersession requires a separate multi-page op.
      if (!c.embedding || c.visibility !== input.visibility || c.expired_at ||
        c.valid_until && new Date(c.valid_until).getTime() <= Date.now() ||
        (input.page_slug ? c.source_markdown_slug !== input.page_slug : c.source_markdown_slug && c.source_markdown_slug !== input.entity_slug) ||
        input.slot && c.claim_metric && !sameClaimSlot(input.slot, c)) continue;
      const next = cosineSimilarity(embedding, c.embedding);
      if (next > score) { score = next; candidate = c; }
    }
    if (candidate && cosineVerdict(lane, score, input.fact, candidate.fact, (await readSupersessionThreshold(engine, embeddingModel, embedding.length)).threshold) === 'duplicate') return {
      status: candidate.kind === input.kind ? 'superseded' : 'duplicate', candidate,
    };
  }
  return { status: 'inserted', candidate: null };
}

/** #6764: a conversation-derived fact changes when its page is re-extracted (forget + remember would only withdraw it). */
function replacesDerivedSuggestion(sourceId: string, slug: string): string {
  const base = `gbrain extract-conversation-facts --source-id ${sourceId} --slugs ${slug} --force`;
  return `Re-extract the conversation page to change its facts: preview with ${base} --dry-run, then run ${base} --max-cost-usd <n> (paid).`;
}

/**
 * #7248: a replacement whose visibility the caller did not set keeps a
 * private target private, so replacing a private fact needs no flag. Only a
 * local caller qualifies (a remote caller never sees a private fact), and a
 * world target never widens a default.
 */
export async function replacementVisibility(engine: BrainEngine, sourceId: string, targetId: number,
  defaulted: 'world' | 'private', opts: { remote: boolean }): Promise<'world' | 'private'> {
  if (defaulted === 'private' || opts.remote || !Number.isSafeInteger(targetId)) return defaulted;
  const [target] = await engine.executeRaw<{ visibility: string }>('SELECT visibility FROM facts WHERE id=$1 AND source_id=$2', [targetId, sourceId]);
  return target?.visibility === 'private' ? 'private' : defaulted;
}

/**
 * `remember.replaces`: the caller names the fact the new one replaces. Checked,
 * zero-model: the target must be an active fact in this source with the same
 * visibility and entity, and on the same entity page; equal text is a no-op
 * duplicate. `lock` takes the target row lock inside the publishing transaction.
 */
export async function decideReplacement(engine: BrainEngine, sourceId: string, input: SingleFactIntent, targetId: number,
  opts: { pageSlug: string; remote: boolean; lock?: boolean; excludePrivate?: boolean; slugPrefixes?: string[] | null }): Promise<FactDecision> {
  // GST-359: a member's write may replace a fact on its own member page (the page this write goes to), private or not.
  const ownPage = input.page_slug && input.page_slug === opts.pageSlug ? input.page_slug : null;
  const notFound = () => verbError('not_found', `No fact with id "${targetId}" to replace.`,
    'Pass the fact_id of an active fact from recall (facts[].fact_id) about the same entity, or omit replaces to save a new fact.');
  const [target] = await engine.executeRaw<FactCandidate & { withdrawn: boolean }>(`SELECT f.*,
      EXISTS (SELECT 1 FROM fact_withdrawals w WHERE w.source_id=f.source_id AND w.visibility=f.visibility
        AND w.fact_hash IN (gbrain_fact_fingerprint(f.fact),gbrain_fact_fingerprint_v1(f.fact))
        AND ${withdrawalCoversFactSql('w', 'f')}) AS withdrawn
    FROM facts f WHERE f.id=$1 AND f.source_id=$2${!opts.excludePrivate ? '' : ownPage ? ` AND (f.source_markdown_slug = $3 OR ${remoteProvenanceFloorSql('f')})` : ` AND ${remoteProvenanceFloorSql('f')}`}${opts.lock ? ' FOR UPDATE OF f' : ''}`,
    opts.excludePrivate && ownPage ? [targetId, sourceId, ownPage] : [targetId, sourceId]);
  if (!target || (opts.remote && target.visibility !== 'world' && !(ownPage && target.source_markdown_slug === ownPage))) throw notFound();
  // #5913 [R2]: a slug-bound writer learns nothing about a fact outside its prefixes; the answer is the missing-id one.
  if (opts.slugPrefixes && [target.entity_slug, target.source_markdown_slug].some(slug => slug && !slugUnderBoundPrefixes(opts.slugPrefixes!, slug))) throw notFound();
  if (target.visibility !== input.visibility) {
    throw verbError('invalid_params', `replaces_visibility_mismatch: fact #${targetId} is ${target.visibility}, and the new fact would be ${input.visibility}.`,
      `Pass visibility: "${target.visibility}" to replace it, or forget fact #${targetId} and remember the new claim separately.`);
  }
  const candidate: FactCandidate = { ...target, id: Number(target.id) };
  if (target.withdrawn) {
    throw verbError('invalid_params', `target_withdrawn: fact #${targetId} was forgotten, so it cannot be replaced.`,
      'Remember the new claim without replaces; the forgotten claim stays withdrawn.');
  }
  if (target.expired_at || (target.valid_until && new Date(target.valid_until).getTime() <= Date.now())) {
    if (target.superseded_by != null) {
      throw verbError('invalid_params', `target_superseded: fact #${targetId} was already replaced by fact #${target.superseded_by}.`,
        `Recall the entity to check the current fact, then pass replaces: "${target.superseded_by}" if the new claim replaces that one.`);
    }
    throw verbError('invalid_params', `target_expired: fact #${targetId} is no longer active.`,
      'Remember the new claim without replaces.');
  }
  // R1: replacing a fact saved without an entity by the same claim with an entity links it (supersession, no withdrawal).
  if (target.entity_slug != null && target.entity_slug !== (input.entity_slug ?? null)) {
    throw verbError('invalid_params', `replaces_entity_mismatch: fact #${targetId} is about ${target.entity_slug ?? 'no entity'}, not ${input.entity_slug ?? 'no entity'}.`,
      'Pass the same entity as the fact being replaced, or forget the old fact and remember the new one separately.');
  }
  // GBRA-190 G4: a fact saved without an entity is a row of memory/unattributed; linking it strikes that row in the same write.
  // GST-2492: so does replacing a fact a strict source held there for the same entity (checked above).
  const linking = input.entity_slug != null && isHoldingSlug(candidate.source_markdown_slug);
  if (candidate.source_markdown_slug && candidate.source_markdown_slug !== opts.pageSlug && !linking) {
    const derived = String(candidate.source ?? '').startsWith('cli:extract-conversation-facts');
    throw verbError('invalid_params', `replaces_cross_page: fact #${targetId} lives on page ${candidate.source_markdown_slug}${derived ? ', extracted from that conversation page' : ''}.`,
      derived ? replacesDerivedSuggestion(sourceId, candidate.source_markdown_slug) : 'Forget the old fact, then remember the new one.');
  }
  const exact = await decideSingleFact(engine, sourceId, input, null, undefined, undefined, { excludePrivate: opts.excludePrivate });
  if (exact.status === 'duplicate') {
    if (exact.candidate!.id === candidate.id) return { status: 'duplicate', candidate };
    throw verbError('invalid_params', `replaces_duplicate: an active fact (#${exact.candidate!.id}) already states this claim.`,
      `Forget fact #${targetId} if it is no longer true; the existing fact #${exact.candidate!.id} already says the new claim.`);
  }
  return { status: 'superseded', candidate };
}