Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
22.4 KB
Raw Blame
import type { BrainEngine } from '../engine.ts';
import { renderFactsTable, type ParsedFact } from '../facts-fence.ts';
import { OperationError, opError, type WithdrawnDetail } from '../ops/contract.ts';
import { fenceOperationError } from '../fence-repair/refusal.ts';
import { withdrawnFact, withdrawalFenceBlocks } from './withdrawal-overlay.ts';
import { ambiguousFenceClaims, discoverWithdrawalTargets, withdrawalDiscoveryFailure } from './withdrawal-discovery.ts';
import { dropPurgedFenceRows } from './purge-overlay.ts';
import { remoteProvenanceFloorSql } from '../search/private-visibility.ts';
import { outcomeMarkerSql } from './audit-sources.ts';
import { coverageActive, noteBelief, readLedgerSource, withBeliefBatch } from '../belief/ledger.ts';
import { createPageVersion } from '../page-state/versions.ts';
import { validateSlug } from '../utils.ts';
import { factWithdrawalSubjectSql, onOwnedMemoryPageSql } from '../access/member-facts.ts';
import { ownedMemorySubjectSql, unbackedEntitySql } from './withdrawal-schema.ts';

export interface WithdrawalCommit {
  withdrawn: boolean;
  pages: Array<{ sourceId: string; slug: string; revision: string }>;
}

export const FACT_WITHDRAWN_MESSAGE = 'fact_withdrawn: this exact claim was explicitly forgotten in this source and visibility.';
/**
 * R1: the refusal names why repeating the claim fails and the write that works. A claim forgotten
 * without an entity is withdrawn for every entity, so "forget, then remember it with an entity" is refused.
 */
export const FACT_WITHDRAWN_SUGGESTION = 'A forgotten claim stays withdrawn, and remembering the same text again is refused; '
  + 'a claim forgotten without an entity is withdrawn for every entity. If it is still true, remember it in new words. '
  + 'To link a fact saved without an entity, do not forget it: remember it with entity and replaces set to its fact id.';

/** DB-first: no filesystem ownership, provider work or root lock is required. */
/**
 * Called inside the withdrawal transaction after the ledger row commits with
 * it: projections derived from the withdrawn claim (beyond pages and chunks,
 * which this module invalidates) register here to invalidate in the same unit.
 */
export type WithdrawalInvalidation = (tx: BrainEngine, w: { sourceId: string; factId: number; subject: string; pages: WithdrawalCommit['pages'] }) => Promise<void>;
const withdrawalInvalidations: WithdrawalInvalidation[] = [];
export function registerWithdrawalInvalidation(fn: WithdrawalInvalidation): void {
  if (!withdrawalInvalidations.includes(fn)) withdrawalInvalidations.push(fn);
}

/** The DB-plane switch for overnight semantic withdrawal review (decide review lane). */
export const REVIEW_WITHDRAW_KEY = 'decide.slots.conflict.review_withdraw';

/**
 * `decide.slots.conflict.review_withdraw`: on unless explicitly turned off. The
 * held-out qualification passed (docs/eval/decisions/p8/SEALED_VERDICTS.md); the
 * lane still proposes only where the conflict slot is on with a decision provider.
 */
export function reviewWithdrawOn(raw: string | null | undefined): boolean {
  const v = (raw ?? '').trim().toLowerCase();
  return v === '' || ['true', 'on', '1', 'yes'].includes(v);
}

/**
 * GST-359 D: the facts a withdrawal admits besides world facts for a remote caller: an attested person's own
 * member-page fact (`memberPage`, positional $4; the caller checked it). Positional $3 is worldOnly.
 */
function withdrawalAdmitSql(excludePrivate?: boolean): string {
  return `AND ($3::boolean=false OR visibility='world' OR (visibility='private' AND source_markdown_slug=$4::text))${excludePrivate ? ` AND ${remoteProvenanceFloorSql('facts')}` : ''}`;
}

/**
 * What withdrawing fact `id` changes: its claim (visibility, fingerprint, subject), whether a withdrawal already
 * covers it, the pages discovery finds carrying the claim for that subject (`affected`), and every page key the
 * withdrawal holds (`slugs`: those pages and the belief streams of the facts it expires). Null when the fact is not
 * visible to the caller. Read-only; forget's preparation (persistence/forget-prepare.ts) and the withdrawal share it.
 * A fact on a member page is withdrawn for that page alone (its subject is the page's slug, GST-359 D).
 */
export async function planFactWithdrawal(engine: BrainEngine, id: number, sourceId: string, worldOnly: boolean, excludePrivate?: boolean, memberPage?: string | null) {
  const visible = await engine.executeRaw<{ visibility: 'private' | 'world'; fact: string; fact_hash: string; subject: string; expired: boolean }>(
    `SELECT visibility,fact,gbrain_fact_fingerprint(fact) AS fact_hash,${factWithdrawalSubjectSql('facts')} AS subject,expired_at IS NOT NULL AS expired FROM facts WHERE id=$1 AND source_id=$2
      AND NOT ${outcomeMarkerSql('facts')} ${withdrawalAdmitSql(excludePrivate)}`, [id, sourceId, worldOnly, memberPage ?? null]);
  if (!visible.length) return null;
  const { visibility, fact, fact_hash, subject, expired } = visible[0];
  // Discovery is keyed on the claim and its subject: a subject-scoped
  // withdrawal changes only that entity's pages and its facts' provenance.
  // Only an existing withdrawal covering this subject makes it a no-op.
  const existing = (await engine.executeRaw(`SELECT 1 FROM fact_withdrawals
    WHERE source_id=$1 AND visibility=$2 AND fact_hash IN ($3,gbrain_fact_fingerprint_v1($5)) AND (subject='*' OR subject=$4) LIMIT 1`,
  [sourceId, visibility, fact_hash, subject, fact])).length > 0;
  const affected = existing ? [] : await discoverWithdrawalTargets(engine, sourceId, [{ visibility, fact_hash, subject, claim: fact }]).catch(withdrawalDiscoveryFailure);
  // The belief ledger's streams of the facts this withdrawal expires (fence pages and database-only subjects), locked with the pages.
  const streams = (await engine.executeRaw<{ slug: string }>(`SELECT DISTINCT COALESCE(source_markdown_slug, entity_slug) AS slug FROM facts
    WHERE source_id=$1 AND visibility=$2 AND gbrain_fact_fingerprint(fact)=$3 AND ($4='*' OR entity_slug=$4 OR (${unbackedEntitySql('facts')} AND NOT ${ownedMemorySubjectSql('$4::text')}) OR (${onOwnedMemoryPageSql('facts')} AND source_markdown_slug=$4)) AND expired_at IS NULL
      AND COALESCE(source_markdown_slug, entity_slug) IS NOT NULL`, [sourceId, visibility, fact_hash, subject]))
    .map(r => r.slug).filter(slug => { try { validateSlug(slug); return true; } catch { return false; } });
  const slugs = [...new Set([...affected.map(page => page.slug), ...streams])].sort();
  return { visibility, fact, fact_hash, subject, expired: expired === true, existing, affected, slugs };
}

export async function recordFactWithdrawal(
  engine: BrainEngine, id: number, sourceId: string, worldOnly = false,
  opts: { requestId?: string; publicationId?: string; semanticReview?: boolean; excludePrivate?: boolean; beliefReport?: { appended: number }; memberPage?: string;
    /**
     * GBRA-190 G4: the withdrawal is the database half of a coordinated publication (forget-prepare.ts) whose page
     * files the coordinator already published and committed, under page keys and a belief batch the publication holds.
     * Nothing is locked, batched or queued here; the ledger row carries the timestamp the files were struck with, and
     * a discovery that no longer finds exactly `expectedSlugs` refuses `revision_conflict` so the request prepares again.
     */
    publication?: { withdrawnAt: string; expectedSlugs: readonly string[] } } = {},
): Promise<WithdrawalCommit> {
  const admit = withdrawalAdmitSql(opts.excludePrivate);
  const pageScoped = `(${onOwnedMemoryPageSql('facts')} AND source_markdown_slug=$4)`;
  return engine.transaction(async tx => {
    // A managed caller takes this EXCLUSIVE source lock before authority,
    // counters and request rows. Repeating an already-held lock is safe.
    if (!opts.publication) await tx.executeRaw('SELECT id FROM sources WHERE id=$1 FOR UPDATE', [sourceId]);
    const plan = await planFactWithdrawal(tx, id, sourceId, worldOnly, opts.excludePrivate, opts.memberPage);
    if (!plan) return { withdrawn: false, pages: [] };
    const { affected, slugs } = plan;
    if (opts.publication) {
      if (JSON.stringify(slugs) !== JSON.stringify([...opts.publication.expectedSlugs].sort())) {
        throw new OperationError('revision_conflict', 'The pages this withdrawal changes moved during preparation.',
          'Read the fact again and submit the forget with a new request_id; nothing was withdrawn.');
      }
    } else await tx.lockPageKeys(slugs.map(slug => ({ sourceId, slug })));
    const rows = await tx.executeRaw<{ visibility: string; fact: string; subject: string }>(
      `SELECT visibility,fact,${factWithdrawalSubjectSql('facts')} AS subject FROM facts WHERE id=$1 AND source_id=$2
        ${admit} FOR UPDATE`, [id, sourceId, worldOnly, opts.memberPage ?? null]);
    if (!rows.length) return { withdrawn: false, pages: [] };
    const row = rows[0];
    const ledger = affected.length ? await readLedgerSource(tx, sourceId) : null;
    // W1 (docs/designs/BELIEF_LEDGER.md section 10): the pre-withdrawal body stays readable at its revision.
    if (ledger && coverageActive(ledger.coverage)) {
      for (const page of affected) {
        const preimage = await tx.readPageSnapshot(page.slug, { sourceId, includeDeleted: true });
        if (preimage) await createPageVersion(tx, page.slug, sourceId, preimage);
      }
    }
    const batch = async (fn: () => Promise<WithdrawalCommit>) => opts.publication ? fn()
      : withBeliefBatch(tx, { sourceId, slugs, publicationId: opts.requestId ?? opts.publicationId ?? null, op: 'forget', pageEventType: 'page.view_changed',
        remote: worldOnly, ...(opts.beliefReport ? { report: opts.beliefReport } : {}) }, fn);
    return batch(async () => {
      // Scoped to the forgotten row's entity: the same claim about another
      // entity stays active and rememberable. A subjectless fact withdraws
      // source-wide ('*'), and so does one on an unbacked name key (GBRA-574),
      // whose rows every withdrawal of the claim reaches, except one on an owned memory page.
      const inserted = await tx.executeRaw(`INSERT INTO fact_withdrawals(source_id,visibility,subject,fact_hash,withdrawn_at)
        VALUES ($1,$2,$4,gbrain_fact_fingerprint($3),COALESCE($5::timestamptz,now())) ON CONFLICT DO NOTHING RETURNING fact_hash`, [sourceId,row.visibility,row.fact,row.subject,opts.publication?.withdrawnAt ?? null]);
      const expired = await tx.executeRaw<{ id: number }>(`UPDATE facts SET expired_at=now(),valid_until=LEAST(COALESCE(valid_until,now()),now())
        WHERE source_id=$1 AND visibility=$2 AND gbrain_fact_fingerprint(fact)=gbrain_fact_fingerprint($3)
          AND ($4='*' OR entity_slug=$4 OR (${unbackedEntitySql('facts')} AND NOT ${ownedMemorySubjectSql('$4::text')}) OR ${pageScoped}) AND expired_at IS NULL RETURNING id`, [sourceId,row.visibility,row.fact,row.subject]);
      noteBelief({ kind: 'withdrawn', factIds: expired.map(r => Number(r.id)) });
      if (!inserted.length) return { withdrawn: false, pages: [] };
      // Overnight semantic review (decide review lane): queued with the ledger row so a late commit is never skipped.
      // Not queued when the caller opted out (`semantic_review: false`, also used by review-accepted withdrawals).
      if (opts.semanticReview !== false && reviewWithdrawOn(await tx.getConfig(REVIEW_WITHDRAW_KEY))) {
        await tx.executeRaw(`INSERT INTO decide_review_queue(kind,source_id,a_ref) VALUES ('withdraw',$1,$2) ON CONFLICT DO NOTHING`, [sourceId, String(id)]);
      }
      // Wave B (spec 11): a segmented page's withdrawn rows are struck in the segment holding the fence, a new
      // generation whose flip retires only the chunks of the occurrences it rewrote.
      const segmented = affected.length ? await tx.executeRaw<{ id: number; slug: string }>(`SELECT id,slug FROM pages
        WHERE source_id=$1 AND slug=ANY($2::text[]) AND body_storage='segmented'`, [sourceId, affected.map(page => page.slug)]) : [];
      if (segmented.length) {
        const { rewriteLiveSegmentedFences } = await import('../page-body/segment-rewrite.ts');
        for (const page of segmented) await rewriteLiveSegmentedFences(tx, Number(page.id), fence => preserveWithdrawnFenceRows(tx, sourceId, fence, page.slug));
      }
      // Logical revision and projection invalidation commit with the withdrawal.
      // The revision trigger queues durable rebuild work even for unmanaged calls.
      const inline = affected.length ? await tx.executeRaw<{ id: number; slug: string; knowledge_revision: string }>(
        `UPDATE pages SET knowledge_revision=gen_random_uuid(),text_projection_revision=NULL,embedding_signature=NULL
          WHERE source_id=$1 AND slug=ANY($2::text[]) AND body_storage<>'segmented' RETURNING id,slug,knowledge_revision`, [sourceId, affected.map(page => page.slug)]) : [];
      if (inline.length) await tx.executeRaw('DELETE FROM content_chunks WHERE page_id=ANY($1::integer[])', [inline.map(page => page.id)]);
      const pages = [...inline, ...segmented.length ? await tx.executeRaw<{ id: number; slug: string; knowledge_revision: string }>(
        'SELECT id,slug,knowledge_revision FROM pages WHERE id=ANY($1::int[])', [segmented.map(page => page.id)]) : []];
      if (opts.requestId && !opts.publication) {
        await tx.executeRaw(`INSERT INTO persistence_effects(request_id,kind,data,source_id,source_incarnation,worktree_id)
          SELECT $1::uuid,k.kind,$3::text::jsonb,s.id,s.incarnation,b.worktree_id
          FROM sources s LEFT JOIN persistence_source_bindings b ON b.source_id=s.id AND b.source_incarnation=s.incarnation
          CROSS JOIN (VALUES ('withdrawal-mirror'),('git'),('embedding')) AS k(kind)
          WHERE s.id=$2 ON CONFLICT(request_id,kind) DO NOTHING`, [opts.requestId, sourceId, JSON.stringify({ version: 2, targets: pages.map(page => ({ slug: page.slug, page_id: page.id, revision: page.knowledge_revision })).sort((a, b) => a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0) })]);
      }
      const committed = pages.map(page => ({ sourceId, slug: page.slug, revision: page.knowledge_revision }));
      for (const invalidate of withdrawalInvalidations) await invalidate(tx, { sourceId, factId: id, subject: row.subject, pages: committed });
      return { withdrawn: true, pages: committed };
    });
  });
}

// Fingerprint each incoming claim once. Inside the join condition the planner
// re-evaluates both fingerprint functions for every (claim, withdrawal) pair,
// so a 28-row fence against 400 withdrawals spent 4 s in this query.
const FINGERPRINTED_CLAIMS = 'SELECT i.visibility, gbrain_fact_fingerprint(i.claim) AS fp, gbrain_fact_fingerprint_v1(i.claim) AS fp_v1';

async function ambiguousFenceMatchesWithdrawal(engine: BrainEngine, sourceId: string, bodies: readonly string[], subject: string | null): Promise<boolean> {
  const claims = bodies.flatMap(ambiguousFenceClaims);
  if (!claims.length) return false;
  const rows = await engine.executeRaw(`WITH incoming AS MATERIALIZED (${FINGERPRINTED_CLAIMS}
      FROM jsonb_to_recordset($2::text::jsonb) i(claim text,visibility text))
    SELECT 1 FROM incoming
    JOIN (SELECT source_id,visibility,subject,fact_hash FROM fact_withdrawals WHERE source_id=$1
      UNION ALL SELECT source_id,visibility,subject,fact_hash FROM fact_purges WHERE source_id=$1) w
      ON w.source_id=$1 AND w.fact_hash IN (incoming.fp,incoming.fp_v1)
      AND (incoming.visibility IS NULL OR w.visibility=incoming.visibility)
      AND ($3::text IS NULL OR w.subject = '*' OR w.subject = $3::text) LIMIT 1`,
  [sourceId, JSON.stringify(claims), subject]);
  return rows.length > 0;
}

async function withdrawalDates(engine: BrainEngine, sourceId: string, facts: readonly ParsedFact[], subject: string | null): Promise<Map<number,string>> {
  if (!facts.length) return new Map();
  const rows = await engine.executeRaw<{ row_num: number; withdrawn_at: string }>(
    `WITH incoming AS MATERIALIZED (${FINGERPRINTED_CLAIMS}, i.row_num
        FROM jsonb_to_recordset($2::text::jsonb) AS i(row_num integer,claim text,visibility text))
      SELECT incoming.row_num, min(w.withdrawn_at)::text AS withdrawn_at FROM incoming
      JOIN fact_withdrawals w ON w.source_id=$1 AND w.visibility=incoming.visibility
        AND w.fact_hash IN (incoming.fp,incoming.fp_v1)
        AND ($3::text IS NULL OR w.subject = '*' OR w.subject = $3::text)
      GROUP BY incoming.row_num`,
    [sourceId, JSON.stringify(facts.map(f => ({ row_num:f.rowNum, claim:f.claim, visibility:f.visibility }))), subject],
  );
  return new Map(rows.map(r => [r.row_num, new Date(r.withdrawn_at).toISOString().slice(0,10)]));
}

/**
 * Overlay stale source files before hashing/chunking, retaining an explicit
 * retraction. Fence rows belong to the page's entity, so pass the page slug
 * as `subject`; without it every subject's withdrawal applies (conservative).
 */
export async function preserveWithdrawnFenceRows(engine: BrainEngine, sourceId: string, body: string, subject?: string): Promise<string> {
  body = await dropPurgedFenceRows(engine, sourceId, body, subject);
  if (!body.includes('gbrain:facts:begin')) return body;
  const blocks = withdrawalFenceBlocks(body);
  for (const block of blocks.reverse()) {
    // Preserve malformed-fence diagnostics; never re-render a partial parse.
    if (block.parsed.warnings.length) continue;
    const dates = await withdrawalDates(engine, sourceId, block.parsed.facts, subject ?? null);
    if (!dates.size) continue;
    const facts = block.parsed.facts.map(f => {
      const date = dates.get(f.rowNum);
      return date ? withdrawnFact(f, date) : f;
    });
    body = body.slice(0, block.start) + renderFactsTable(facts) + body.slice(block.end);
  }
  return body;
}

/**
 * Refuse provider-free preparation that raced a newly committed withdrawal.
 * `subject` is the page slug the prepared body belongs to (see
 * preserveWithdrawnFenceRows); without it every subject's withdrawal counts.
 */
export async function assertPreparedFactWithdrawals(engine: BrainEngine, sourceId: string, body: string, timeline: string, subject?: string): Promise<void> {
  const changed = await preserveWithdrawnFenceRows(engine, sourceId, body, subject) !== body ||
    await preserveWithdrawnFenceRows(engine, sourceId, timeline, subject) !== timeline;
  const blocked = await ambiguousFenceMatchesWithdrawal(engine, sourceId, [body, timeline], subject ?? null);
  if (changed) {
    throw new OperationError('revision_conflict', 'A fact withdrawal changed during import preparation. Retry the import.',
      'Read the current page revision, then submit the updated import with a new request_id.');
  }
  if (blocked) {
    // Typed invalid_fence, wire invalid_params (E6): a managed sync holds this file instead of blocking.
    const section = await ambiguousFenceMatchesWithdrawal(engine, sourceId, [body], subject ?? null) ? 'body' : 'timeline';
    throw fenceOperationError({ reason: 'withdrawn_claim_in_malformed_fence', fence: 'facts', section, rows: [], columns: [], line: null }, subject, sourceId);
  }
}

/**
 * Explicit remember is not an implicit restore operation for that entity. A purged claim counts as withdrawn for every writer.
 * A null `visibility` matches a withdrawal or purge at any visibility: automatic writers never admit a claim forgotten at
 * another visibility, so narrowing the default visibility cannot bring it back.
 */
export async function isFactWithdrawn(
  engine: BrainEngine, sourceId: string, visibility: string | null, claim: string, entitySlug: string | null, memberPage: string | null = null,
): Promise<boolean> {
  // GST-359 D: `memberPage` (a claim saved on a member page) also matches that page's own withdrawals.
  const rows = await engine.executeRaw(`SELECT 1 FROM fact_withdrawals
    WHERE source_id=$1 AND ($2::text IS NULL OR visibility=$2) AND fact_hash IN (gbrain_fact_fingerprint($3),gbrain_fact_fingerprint_v1($3))
      AND (subject = '*' OR subject = $4::text OR subject = $5::text)
    UNION ALL SELECT 1 FROM fact_purges WHERE source_id=$1 AND ($2::text IS NULL OR visibility=$2) AND fact_hash=gbrain_fact_fingerprint($3)
      AND (subject = '*' OR subject = $4::text) LIMIT 1`, [sourceId,visibility,claim,entitySlug,memberPage]);
  return rows.length > 0;
}

/** #5575: the claim matches a purge tombstone (the facts guard raises the same typed code on insert). */
export async function isFactPurged(engine: BrainEngine, sourceId: string, visibility: string, claim: string, entitySlug: string | null): Promise<boolean> {
  const rows = await engine.executeRaw(`SELECT 1 FROM fact_purges WHERE source_id=$1 AND visibility=$2 AND fact_hash=gbrain_fact_fingerprint($3)
    AND (subject = '*' OR subject = $4::text) LIMIT 1`, [sourceId, visibility, claim, entitySlug]);
  return rows.length > 0;
}

/** Writers check this before isFactWithdrawn so a purged claim refuses with typed purged_content, before any provider work. */
export async function assertFactNotPurged(engine: BrainEngine, sourceId: string, input: { visibility: string; fact: string; entity_slug: string | null }): Promise<void> {
  if (!await isFactPurged(engine, sourceId, input.visibility, input.fact, input.entity_slug)) return;
  throw opError('purged_content', 'purged_content: this claim was purged from this source and cannot be saved again.',
    'Purged content stays out of the brain. If it is still true, remember it in new words; only the owner can clear a purge tombstone.');
}

export const WITHDRAWN_DETAIL_VERSION = 1;
/** The bounded, versioned `error_detail` a `fact_withdrawn` refusal stores, so a failed receipt keeps its `withdrawn` object. */
export interface WithdrawalFailureDetail { origin: 'withdrawal'; withdrawn: WithdrawnDetail & { version: typeof WITHDRAWN_DETAIL_VERSION } }

export function withdrawalFailureDetail(error: Pick<OperationError, 'withdrawn'>): WithdrawalFailureDetail | undefined {
  return error.withdrawn ? { origin: 'withdrawal', withdrawn: { version: WITHDRAWN_DETAIL_VERSION, ...error.withdrawn } } : undefined;
}

/** The `withdrawn` object a stored `error_detail` carries, validated field by field, or undefined. */
export function withdrawnFromDetail(detail: unknown): WithdrawnDetail | undefined {
  if (!detail || typeof detail !== 'object' || (detail as { origin?: unknown }).origin !== 'withdrawal') return undefined;
  const w = (detail as { withdrawn?: Record<string, unknown> }).withdrawn;
  if (!w || w.version !== WITHDRAWN_DETAIL_VERSION || typeof w.at !== 'string' || !Number.isFinite(Date.parse(w.at))) return undefined;
  if (typeof w.subject !== 'string' || !w.subject || w.subject.length > 500 || (w.visibility !== 'world' && w.visibility !== 'private')) return undefined;
  return { at: w.at, subject: w.subject, visibility: w.visibility };
}