Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
22.7 KB
Raw Blame
import type { BrainEngine, LinkBatchInput } from './engine.ts';
import { assertPageRevision, PageRevisionConflictError, RevisionBackfillPendingError, type PageSnapshot } from './page-state/types.ts';
import { pipelined } from './page-state/transactions.ts';
import { PageSourceMissingError } from './page-state/guards.ts';
import { beliefCoveredSources, unmanagedBeliefPoint } from './belief/projection-point.ts';
import { executeRawJsonb } from './sql-query.ts';
import { pageSnapshotKey } from './page-snapshot-batch.ts';
import { sanitizeForJsonb } from './batch-rows.ts';
import { replaceWantedLinks, type WantedLinksReplacement } from './wanted-links-store.ts';
import { applyTemporalEvidence, relationshipKeysForOrigin, type TemporalBody } from './link-temporal-apply.ts';
import { primeRelationSemantics } from './link-semantics-pack.ts';
import { effectiveRangesEnabled, LINK_EXTRACTION_GENERATION_KEY, type LineGrammarSettings } from './line-grammar.ts';

export interface DerivedLinkOrigin {
  slug: string;
  sourceId: string;
  expectedRevision: string;
  sourceIncarnation: string;
  /**
   * The caller's own read of the origin (live rows only) in the transaction this
   * replacement nests in, taken under that transaction's guard of every key the
   * replacement locks and after its last write to the origin; skips the re-read.
   */
  snapshot?: PageSnapshot | null;
}

export interface DerivedLinkReplacementOptions {
  includeFrontmatter?: boolean;
  preserveExisting?: boolean;
  includeLegacyNullProducer?: boolean;
  expectedEndpoints?: Array<{ slug: string; sourceId: string; revision: string }>;
  /** The origin's unresolved authored references, replaced in the same transaction (wanted pages). */
  wanted?: WantedLinksReplacement;
  /** The caller installed the pack relation semantics already (links-preparation.ts `primeSemantics`). */
  semanticsPrimed?: boolean;
  /**
   * The line-grammar settings the links were prepared under. Publication uses
   * them for validity ranges (never a second read) and refuses with
   * DerivedLinkSettingsChangedError when the extraction generation moved since,
   * so a page is never published from one setting and ranged from another.
   */
  lineGrammar?: LineGrammarSettings;
  /** The origin body's temporal inputs, read at preparation; a segmented origin's body is never read inside the publication. */
  temporalBody?: TemporalBody;
}

export class DerivedLinkRepairRequiredError extends Error {
  readonly code = 'derived_link_provenance_required';
  constructor() {
    super('Derived frontmatter edges have no origin. Repair their provenance before reconciliation.');
    this.name = 'DerivedLinkRepairRequiredError';
  }
}

export class DerivedLinkEndpointChangedError extends Error {
  readonly code = 'revision_conflict';
}

/** The line-grammar settings changed between preparation and publication; the page stays stale and re-extracts. */
export class DerivedLinkSettingsChangedError extends DerivedLinkEndpointChangedError {
  constructor() { super('Line-grammar settings changed after these links were prepared'); this.name = 'DerivedLinkSettingsChangedError'; }
}

/** replaceDerivedLinks, or null when the line-grammar settings moved since preparation (the page stays stale). */
export async function replaceDerivedLinksUnlessSettingsChanged(engine: Pick<BrainEngine, 'replaceDerivedLinks'>,
  ...args: Parameters<BrainEngine['replaceDerivedLinks']>): Promise<{ created: number; removed: number } | null> {
  try { return await engine.replaceDerivedLinks(...args); }
  catch (error) { if (error instanceof DerivedLinkSettingsChangedError) return null; throw error; }
}

/**
 * #6272: a concurrent writer moved a link endpoint after type resolution, or the origin itself after its snapshot
 * read. Instance checks only: settings changes, backfill-pending revisions and every other error stay fatal.
 */
export function isConcurrentLinkWriteConflict(error: unknown): boolean {
  return (error instanceof DerivedLinkEndpointChangedError && !(error instanceof DerivedLinkSettingsChangedError))
    || (error instanceof PageRevisionConflictError && !(error instanceof RevisionBackfillPendingError));
}

/**
 * A deferring caller's outcome for one origin: written, left stale by a settings change or a concurrent write, or
 * (#6808) refused because the origin's source or a link target's source has no sources row (the missing source id).
 */
export type DeferredDerivedLinkOutcome = { created: number; removed: number } | 'settings_changed' | 'concurrent_write' | { unknownSource: string };

/** replaceDerivedLinksUnlessSettingsChanged for a run that defers (and counts) pages a concurrent writer moved or whose source is gone. */
export async function replaceDerivedLinksDeferringConflicts(engine: Pick<BrainEngine, 'replaceDerivedLinks'>,
  ...args: Parameters<BrainEngine['replaceDerivedLinks']>): Promise<DeferredDerivedLinkOutcome> {
  try { return await replaceDerivedLinksUnlessSettingsChanged(engine, ...args) ?? 'settings_changed'; }
  catch (error) {
    if (error instanceof PageSourceMissingError) return { unknownSource: error.sourceId };
    if (isConcurrentLinkWriteConflict(error)) return 'concurrent_write';
    throw error;
  }
}

/** #6808: a page a DB link drain deferred because a source it writes through has no sources row. */
export interface UnknownSourceSkip { slug: string; source_id: string; missing_source: string }

/** The operator line for unknown-source deferrals: unlike the lines above, re-running extraction does not clear them. */
export function unknownSourceSkipLine(skips: readonly UnknownSourceSkip[]): string {
  const missing = [...new Set(skips.map(s => s.missing_source))].sort();
  const sample = skips.slice(0, 5).map(s => `${s.source_id}:${s.slug}`).join(', ');
  return `Skipped ${skips.length} page(s) because source(s) ${missing.join(', ')} have no sources row (unknown_source; e.g. ${sample}); ` +
    'their links and watermark were kept and the run is incomplete for them. Re-running extract does not fix this: re-register the source ' +
    '(`gbrain sources list` shows the registered ones) or remove its pages.';
}

/** The operator line for pages a settings change left stale mid-run. */
export const settingsChangedSkipLine = (n: number) =>
  `Skipped ${n} page(s) because the line-grammar settings changed during this run; they stay stale. Run \`gbrain extract --stale\` to finish them.`;

/** The operator line for pages a concurrent write left stale mid-run. */
export const concurrentWriteSkipLine = (n: number) =>
  `Skipped ${n} page(s) another writer changed during this run (the page or a link target); their links and watermark were kept, so they stay stale. Run \`gbrain extract --stale\` to finish them.`;

export async function applyAttendanceDelta(tx: Pick<BrainEngine, 'executeRaw' | 'addLinksBatch'>,
  origin: { id: string; slug: string; source_id: string; type: string }, remove: string[], additions: LinkBatchInput[]) {
  if (origin.type !== 'meeting' || remove.length > 512 || additions.length > 256
    || additions.some(row => row.link_type !== 'attended' || row.to_slug !== origin.slug
      || row.to_source_id !== origin.source_id || row.from_source_id !== origin.source_id
      || row.origin_slug !== origin.slug || row.origin_source_id !== origin.source_id
      || row.link_source !== 'markdown')) throw new Error('Invalid attendance delta');
  const removed = remove.length ? await tx.executeRaw(`DELETE FROM links WHERE id=ANY($1::bigint[])
    AND link_type='attended' AND link_source='markdown'
    AND (origin_page_id=$2::bigint OR (origin_page_id IS NULL AND from_page_id=$2::bigint AND link_source='markdown'))
    RETURNING id`, [remove, origin.id]) : [];
  if (removed.length !== remove.length) throw new Error('Attendance removal changed');
  const created = additions.length ? await tx.addLinksBatch(additions, { auditSite: 'addLinksBatch' }) : 0;
  if (created !== additions.length) throw new Error('Attendance insertion did not persist the approved delta');
  return { created, removed: removed.length };
}

/** Validate and normalize one origin's candidate rows (deduplicated, producer and origin stamped). */
function derivedLinkRows(origin: DerivedLinkOrigin, links: LinkBatchInput[], opts: DerivedLinkReplacementOptions) {
  const producers = ['markdown', 'wikilink-resolved', ...(opts.includeFrontmatter === false ? [] : ['frontmatter'])];
  const unique = new Map<string, LinkBatchInput>();
  for (const link of links) {
    const producer = link.link_source ?? 'markdown';
    if (!producers.includes(producer)) throw new TypeError(`Only selected derived link producers can be replaced (got: ${JSON.stringify(producer)}, allowed: ${producers.join(', ')})`);
    if ((link.origin_slug && link.origin_slug !== origin.slug)
      || (link.origin_source_id && link.origin_source_id !== origin.sourceId)) {
      throw new TypeError('Derived link origin does not match the replacement scope');
    }
    const reversedAttendance = (producer === 'markdown' || producer === 'wikilink-resolved') && link.link_type === 'attended'
      && link.origin_slug === origin.slug && link.origin_source_id === origin.sourceId
      && (link.from_slug !== origin.slug || (link.from_source_id ?? origin.sourceId) !== origin.sourceId)
      && link.to_slug === origin.slug && (link.to_source_id ?? origin.sourceId) === origin.sourceId;
    const row = { ...link, link_source: producer, origin_slug: producer === 'frontmatter' || reversedAttendance ? origin.slug : undefined, origin_source_id: origin.sourceId,
      from_source_id: link.from_source_id ?? origin.sourceId, to_source_id: link.to_source_id ?? origin.sourceId };
    if (producer !== 'frontmatter' && !reversedAttendance && (row.from_slug !== origin.slug || row.from_source_id !== origin.sourceId)) {
      throw new TypeError('Markdown links must originate at the replaced page');
    }
    if (row.from_slug !== origin.slug || row.from_source_id !== origin.sourceId) {
      if (row.to_slug !== origin.slug || row.to_source_id !== origin.sourceId) throw new TypeError('Derived links must reference their origin');
    }
    const key = JSON.stringify([row.from_source_id, row.from_slug, row.to_source_id, row.to_slug, row.link_type ?? '', producer]);
    if (!unique.has(key)) unique.set(key, row);
  }
  return { producers, rows: [...unique.values()] };
}

const derivedLinkLockKeys = (origin: DerivedLinkOrigin, rows: LinkBatchInput[]) => [{ sourceId: origin.sourceId, slug: origin.slug },
  ...rows.flatMap(row => [{ sourceId: row.from_source_id!, slug: row.from_slug }, { sourceId: row.to_source_id!, slug: row.to_slug }])];

/** FOR SHARE: a concurrent setting change (which updates this row) waits for this publication, or this one sees it. */
const lockedExtractionGeneration = async (tx: BrainEngine) => (await tx.executeRaw<{ value: string }>(
  'SELECT value FROM config WHERE key = $1 FOR SHARE', [LINK_EXTRACTION_GENERATION_KEY]))[0]?.value ?? null;

export async function replaceDerivedLinks(
  engine: Pick<BrainEngine, 'transaction' | 'executeRaw'>,
  origin: DerivedLinkOrigin,
  links: LinkBatchInput[],
  opts: DerivedLinkReplacementOptions = {},
): Promise<{ created: number; removed: number }> {
  const { producers, rows } = derivedLinkRows(origin, links, opts);
  const beliefs = unmanagedBeliefPoint(origin.sourceId, [origin.slug], 'derived_links');
  const result = await engine.transaction(async tx => {
    await pipelined(tx, [
      async () => { if (!opts.semanticsPrimed) await primeRelationSemantics(tx); },
      () => tx.lockPageKeys(derivedLinkLockKeys(origin, rows)),
    ]);
    if (opts.lineGrammar && await lockedExtractionGeneration(tx) !== opts.lineGrammar.generation) throw new DerivedLinkSettingsChangedError();
    const snapshot = origin.snapshot !== undefined ? origin.snapshot : await tx.readPageSnapshot(origin.slug, { sourceId: origin.sourceId });
    return beliefs.run(tx, () => publishDerivedLinks(tx, origin, producers, rows, opts, snapshot));
  });
  await beliefs.commit(engine);
  return result;
}

export interface DerivedLinkBatchItem { origin: DerivedLinkOrigin; links: LinkBatchInput[]; opts?: DerivedLinkReplacementOptions }

/**
 * replaceDerivedLinks for many origins in one transaction (bulk `extract
 * --source db`). Each origin runs the same checks and writes as its own call,
 * in order; the relation-semantics prime, the page guards (every origin and
 * endpoint, one sorted lock), the extraction-generation check and the origin
 * reads are shared. Any failure, a settings change included, rolls the whole
 * batch back; callers that need per-origin outcomes then replay the origins
 * one by one through replaceDerivedLinks.
 */
export async function replaceDerivedLinksBatch(
  engine: Pick<BrainEngine, 'transaction' | 'executeRaw'>,
  items: readonly DerivedLinkBatchItem[],
): Promise<Array<{ created: number; removed: number }>> {
  const prepared = items.map(item => ({ origin: item.origin, opts: item.opts ?? {}, ...derivedLinkRows(item.origin, item.links, item.opts ?? {}) }));
  if (!prepared.length) return [];
  const beliefs = prepared.map(item => unmanagedBeliefPoint(item.origin.sourceId, [item.origin.slug], 'derived_links'));
  const results = await engine.transaction(async tx => {
    await pipelined(tx, [
      async () => { if (prepared.some(item => !item.opts.semanticsPrimed)) await primeRelationSemantics(tx); },
      () => tx.lockPageKeys(prepared.flatMap(item => derivedLinkLockKeys(item.origin, item.rows))),
    ]);
    if (prepared.some(item => item.opts.lineGrammar)) {
      const generation = await lockedExtractionGeneration(tx);
      if (prepared.some(item => item.opts.lineGrammar && item.opts.lineGrammar.generation !== generation)) throw new DerivedLinkSettingsChangedError();
    }
    const { snapshots } = await tx.readPageSnapshotsBatch(prepared.filter(item => item.origin.snapshot === undefined)
      .map(item => ({ slug: item.origin.slug, sourceId: item.origin.sourceId })), { maxBytes: Number.MAX_SAFE_INTEGER });
    const covered = await beliefCoveredSources(tx, prepared.map(item => item.origin.sourceId));
    const results: Array<{ created: number; removed: number }> = [];
    for (const [i, { origin, producers, rows, opts }] of prepared.entries()) {
      const snapshot = origin.snapshot !== undefined ? origin.snapshot : snapshots.get(pageSnapshotKey(origin.sourceId, origin.slug)) ?? null;
      const publish = () => publishDerivedLinks(tx, origin, producers, rows, opts, snapshot);
      results.push(await (covered.has(origin.sourceId) ? beliefs[i]!.run(tx, publish) : publish()));
    }
    return results;
  });
  for (const point of beliefs) await point.commit(engine);
  return results;
}

/**
 * Publish through replaceDerivedLinksBatch; when that transaction fails,
 * replay the origins one by one so every outcome is the unbatched one:
 * `settings_changed` for a settings-changed skip, `concurrent_write` for an
 * origin a concurrent writer moved (#6272), `{ unknownSource }` for a missing source (#6808), and any other failing origin's
 * error thrown (after `onError`) with the origins before it committed.
 */
export async function replaceDerivedLinksBatchOrReplay(engine: Pick<BrainEngine, 'replaceDerivedLinks' | 'replaceDerivedLinksBatch'>,
  items: readonly DerivedLinkBatchItem[], onError?: (item: DerivedLinkBatchItem) => void): Promise<DeferredDerivedLinkOutcome[]> {
  if (!items.length) return [];
  try { return await engine.replaceDerivedLinksBatch(items); }
  catch {
    const results: DeferredDerivedLinkOutcome[] = [];
    for (const item of items) {
      try { results.push(await replaceDerivedLinksDeferringConflicts(engine, item.origin, item.links, item.opts)); }
      catch (error) { onError?.(item); throw error; }
    }
    return results;
  }
}

/** Under the caller's guards: check the origin read, then replace its derived links and temporal evidence. */
async function publishDerivedLinks(tx: BrainEngine, origin: DerivedLinkOrigin, producers: string[], rows: LinkBatchInput[],
  opts: DerivedLinkReplacementOptions, snapshot: PageSnapshot | null): Promise<{ created: number; removed: number }> {
  assertPageRevision(snapshot, { expectedRevision: origin.expectedRevision });
  if (!snapshot || snapshot.sourceIncarnation !== origin.sourceIncarnation || snapshot.page.deleted_at) {
    throw new Error('Derived link origin changed or was deleted');
  }
  const id = snapshot.page.id;
  // The wanted-page replacement and every read the checks below need are sent together, then judged in order.
  // Temporal evidence (tense, dated transitions, relationship state) is part
  // of the same derived projection: captured before, replaced after.
  type Existing = { id: number; from_slug: string; to_slug: string; from_source_id: string; to_source_id: string; link_type: string;
    link_source: string | null; origin_slug: string | null; origin_source_id: string | null; context: string; origin_field: string | null };
  const [, temporalKeysBefore, ambiguous, missing, changed, existing, inlineRanges] = await pipelined(tx, [
    async () => { if (opts.wanted) await replaceWantedLinks(tx, { pageId: id, sourceId: origin.sourceId }, opts.wanted); },
    () => relationshipKeysForOrigin(tx, Number(id)),
    () => opts.includeFrontmatter !== false && !opts.preserveExisting ? tx.executeRaw(`SELECT 1 FROM links WHERE link_source='frontmatter'
        AND origin_page_id IS NULL AND (from_page_id=$1 OR to_page_id=$1) LIMIT 1`, [id]) : Promise.resolve([]),
    () => executeRawJsonb(tx, `SELECT 1 FROM jsonb_to_recordset(($1::jsonb)->'rows')
      AS v(from_slug text, to_slug text, from_source_id text, to_source_id text)
      LEFT JOIN pages f ON f.slug=v.from_slug AND f.source_id=v.from_source_id AND f.deleted_at IS NULL
      LEFT JOIN pages t ON t.slug=v.to_slug AND t.source_id=v.to_source_id AND t.deleted_at IS NULL
      WHERE f.id IS NULL OR t.id IS NULL LIMIT 1`, [], [{ rows }]),
    () => opts.expectedEndpoints?.length ? executeRawJsonb(tx, `SELECT 1 FROM jsonb_to_recordset(($1::jsonb)->'rows')
        AS v(slug text, "sourceId" text, revision text)
        LEFT JOIN pages p ON p.slug=v.slug AND p.source_id=v."sourceId" AND p.deleted_at IS NULL
        WHERE p.id IS NULL OR p.knowledge_revision::text <> v.revision LIMIT 1`, [], [{ rows: opts.expectedEndpoints }]) : Promise.resolve([]),
    () => opts.preserveExisting ? tx.executeRaw<Existing>(`SELECT l.id, f.slug from_slug, t.slug to_slug,
          f.source_id from_source_id, t.source_id to_source_id, l.link_type, l.link_source,
          o.slug origin_slug, o.source_id origin_source_id, l.context, l.origin_field
        FROM links l JOIN pages f ON f.id=l.from_page_id JOIN pages t ON t.id=l.to_page_id
        LEFT JOIN pages o ON o.id=l.origin_page_id
        WHERE (l.link_source=ANY($2::text[]) OR ($3::boolean AND l.link_source IS NULL))
          AND (l.origin_page_id=$1 OR (l.origin_page_id IS NULL AND l.from_page_id=$1
            AND (l.link_source IN ('markdown','wikilink-resolved') OR l.link_source IS NULL)))`, [id, producers, opts.includeLegacyNullProducer !== false]) : Promise.resolve([]),
    () => opts.lineGrammar ? Promise.resolve(opts.lineGrammar.effectiveRanges) : effectiveRangesEnabled(tx),
  ]) as [unknown, Awaited<ReturnType<typeof relationshipKeysForOrigin>>, unknown[], unknown[], unknown[], Existing[], boolean];
  const withTemporal = async (result: { created: number; removed: number }) => {
    await applyTemporalEvidence(tx, snapshot.page, rows, temporalKeysBefore, { inlineRanges, body: opts.temporalBody });
    return result;
  };
  if (ambiguous.length) throw new DerivedLinkRepairRequiredError();
  if (missing.length) throw new DerivedLinkEndpointChangedError('A derived link endpoint changed or was deleted');
  if (changed.length) throw new DerivedLinkEndpointChangedError('A derived link endpoint changed after type resolution');
  const reversed = rows.filter(row => row.link_type === 'attended' && row.origin_slug
    && row.to_slug === origin.slug && row.to_source_id === origin.sourceId
    && (row.from_slug !== origin.slug || row.from_source_id !== origin.sourceId));
  if (reversed.length) {
    if (snapshot.page.type !== 'meeting' || reversed.some(row => row.link_source !== 'frontmatter' && !opts.expectedEndpoints?.some(endpoint =>
      endpoint.slug === row.from_slug && endpoint.sourceId === row.from_source_id))) {
      throw new TypeError('Canonical attendance requires a meeting origin and revision-bound person endpoints');
    }
    const invalid = await executeRawJsonb(tx, `SELECT 1 FROM jsonb_to_recordset(($1::jsonb)->'rows')
        AS v(from_slug text, from_source_id text)
        JOIN pages p ON p.slug=v.from_slug AND p.source_id=v.from_source_id
        WHERE p.type <> 'person' LIMIT 1`, [], [{ rows: reversed }]);
    if (invalid.length) throw new TypeError('Canonical attendance requires person endpoints');
  }
  if (opts.preserveExisting) {
    const identity = (row: Pick<LinkBatchInput, 'from_source_id' | 'from_slug' | 'to_source_id' | 'to_slug' | 'link_type'>
      & { link_source?: string | null; origin_slug?: string | null }) => JSON.stringify([row.from_source_id, row.from_slug,
      row.to_source_id, row.to_slug, row.link_type ?? '', row.link_source ?? 'markdown', row.origin_slug ?? null]);
    const wanted = new Map(rows.map(row => [identity(row), row]));
    const retained = new Set(existing.map(identity));
    const obsolete = existing.filter(row => !wanted.has(identity(row))).map(row => row.id);
    if (obsolete.length) await tx.executeRaw('DELETE FROM links WHERE id=ANY($1::bigint[])', [obsolete]);
    const updates = existing.flatMap(previous => {
      const desired = wanted.get(identity(previous));
      if (!desired) return [];
      const context = sanitizeForJsonb(desired.context || '');
      const origin_field = desired.origin_field || null;
      return previous.context !== context || previous.origin_field !== origin_field
        ? [{ id: previous.id, context, origin_field }] : [];
    });
    if (updates.length) await executeRawJsonb(tx, `UPDATE links l SET context=v.context, origin_field=v.origin_field
        FROM jsonb_to_recordset(($1::jsonb)->'rows') AS v(id bigint, context text, origin_field text)
        WHERE l.id=v.id`, [], [{ rows: updates }]);
    const additions = rows.filter(row => !retained.has(identity(row)));
    const created = additions.length ? await tx.addLinksBatch(additions, { auditSite: 'addLinksBatch' }) : 0;
    if (created !== additions.length) throw new Error('Derived link replacement did not persist every candidate');
    return withTemporal({ created, removed: obsolete.length });
  }
  const removed = await tx.executeRaw(`DELETE FROM links WHERE link_source=ANY($2::text[])
      AND (origin_page_id=$1 OR (origin_page_id IS NULL AND from_page_id=$1
        AND link_source IN ('markdown','wikilink-resolved'))) RETURNING id`, [id, producers]);
  const created = await tx.addLinksBatch(rows, { auditSite: 'addLinksBatch' });
  if (created !== rows.length) throw new Error('Derived link replacement did not persist every candidate');
  return withTemporal({ created, removed: removed.length });
}