Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
28.7 KB
Raw Blame
/**
 * Pages that link to an entity, one row per referring page: the entity card's
 * `referenced_by` groups and `get_backlinks {group: "page"}` both read this
 * one helper, so a card group and its continuation agree on membership,
 * order, privacy and identity union.
 *
 * - Every link source counts (mentions included), unlike `backlink_count`.
 * - Rows are ordered by `(date DESC, source_id, slug)` where date is
 *   `COALESCE(effective_date, updated_at)`; paging is a keyset cursor.
 * - Grouping is by the referring page's pack-canonical type (stored type
 *   mapped through the pack's type aliases; untyped pages are `untyped`).
 *   A `type` filter accepts a canonical type or a stored type.
 * - Untrusted callers never see a private or derived-private referrer, nor a
 *   link whose origin page is private, in rows or counts.
 * - With `entity_identity.union` on, links to the page's identity co-members
 *   count as links to the page.
 * - #5575: a quarantined referrer is left out (rows and counts) unless an
 *   authorized `include_quarantined` read asked; the caller's eligibility
 *   (floor, activation control on proactive reads) applies before LIMIT; each
 *   row carries its page's `trust_tier` + `origin` (+ `unconfirmed` when
 *   flagged), which text renderers label the preview with.
 */

import type { BrainEngine } from '../engine.ts';
import { boundedBodyPrefixSql } from '../page-window/bounded-lead.ts';
import { privatePagesFilterFastForCaller, privatePagesFilterForCaller, privateLinkOriginFilterForCaller } from '../search/private-visibility.ts';
import { isIdentityUnionEnabled, listEntityIdentities } from '../entity-identity.ts';
import { stripTakesFence } from '../takes-fence.ts';
import { stripFactsFence } from '../facts-fence.ts';
import { redactFindings } from '../secret-scan.ts';
import { typeAliasPairs, type PackTypes } from './policy.ts';
import { quarantineFilterFragment, isQuarantined } from '../quarantine.ts';
import { activationSuppressedSql, pageEligibleSql } from '../eligibility/sql.ts';
import { trustFields, type TrustFields } from '../eligibility/labels.ts';
import type { ReadEligibility } from '../eligibility/policy.ts';

export const PREVIEW_CHARS = 160;
export const GROUP_ROW_CAP = 10;
export const CARD_ROW_CAP = 50;
export const PAGE_DEFAULT_LIMIT = 50;
export const PAGE_MAX_LIMIT = 500;

export interface ReferenceRow extends TrustFields {
  slug: string;
  source_id: string;
  title: string;
  type: string | null;
  canonical_type: string;
  date: string | null;
  /** `effective_date` provenance (event_date, date, published, filename, fallback) or `updated_at`. */
  date_source: string;
  /** First 160 characters of body text, private fences stripped and secrets redacted. Not evidence: fetch the page. */
  preview: string;
  /** Only on an authorized include_quarantined read: the referrer is quarantined; its preview renders as external data. */
  quarantined?: true;
}

export interface ReferrerScope {
  /** The entity page. */
  slug: string;
  sourceId: string;
  /** Sources a referring page may live in (the caller's read scope). */
  referrerSources: string[];
  excludePrivate: boolean;
  pack: PackTypes | null;
  /** Visibility kept in previews (world only for untrusted callers). */
  keepVisibility: ('private' | 'world')[];
  /** #5575: the caller's read eligibility: rows below the floor, or flagged under activation control, are left out. */
  eligibility?: ReadEligibility;
  /** An authorized include_quarantined read keeps quarantined referrers; every other read leaves them out. */
  includeQuarantined?: boolean;
}

interface RawRow { id: number; slug: string; source_id: string; type: string | null; canonical_type: string;
  d: string | null; gtotal?: number }

/** The entity page plus, under `entity_identity.union`, its identity co-members. */
async function targetPages(engine: BrainEngine, scope: ReferrerScope, union?: boolean): Promise<{ sources: string[]; slugs: string[] }> {
  const targets = [{ slug: scope.slug, source_id: scope.sourceId }];
  if (union ?? await isIdentityUnionEnabled(engine)) {
    try {
      const members = await listEntityIdentities(engine, { slug: scope.slug, slugSourceId: scope.sourceId, sourceId: scope.sourceId,
        allowedSources: scope.referrerSources, excludePrivate: scope.excludePrivate });
      for (const m of members) if (!targets.some(t => t.slug === m.slug && t.source_id === m.source_id)) targets.push({ slug: m.slug, source_id: m.source_id });
    } catch { /* the union never breaks the base read */ }
  }
  return { sources: targets.map(t => t.source_id), slugs: targets.map(t => t.slug) };
}

/**
 * The shared CTE `typed`: one row per referring page with its canonical type
 * and date. Params $1..$5 are fixed (target sources, target slugs, referrer
 * sources, stored type aliases, their canonical types; all bound as text[]).
 * The target's own privacy is checked once in `tgt`, and the referring page
 * ids are collected once in `refs`, so a hub with thousands of inbound links
 * costs one hashed membership test per candidate page.
 */
/** The caller's read filters, shared by the single-target CTE and its batched variant. */
function referrerFilters(scope: Pick<ReferrerScope, 'excludePrivate' | 'eligibility' | 'includeQuarantined'>) {
  return {
    targetPriv: scope.excludePrivate ? ` AND ${privatePagesFilterForCaller('t')}` : '',
    linkPriv: scope.excludePrivate ? ` AND ${privateLinkOriginFilterForCaller('l')}` : '',
    pagePriv: scope.excludePrivate ? ` AND ${privatePagesFilterFastForCaller('f')}` : '',
    pageTrust: ` AND ${pageEligibleSql('f', scope.eligibility)}${scope.includeQuarantined ? '' : ` AND ${quarantineFilterFragment('f')}`}`,
  };
}

function referrersCte(scope: ReferrerScope): string {
  const { targetPriv, linkPriv, pagePriv, pageTrust } = referrerFilters(scope);
  return `WITH tgt AS MATERIALIZED (
      SELECT t.id FROM pages t JOIN unnest($1::text[], $2::text[]) AS k(s, g) ON t.source_id = k.s AND t.slug = k.g
       WHERE t.deleted_at IS NULL${targetPriv}
    ),
    refs AS MATERIALIZED (
      SELECT DISTINCT l.from_page_id AS id FROM links l WHERE l.to_page_id IN (SELECT id FROM tgt)${linkPriv}
    ),
    typed AS MATERIALIZED (
      SELECT f.id, f.slug, f.source_id, f.type,
             COALESCE(f.effective_date, f.updated_at) AS d,
             COALESCE(($5::text[])[array_position($4::text[], f.type)], NULLIF(f.type, ''), 'untyped') AS canonical_type
        FROM pages f
       WHERE f.id IN (SELECT id FROM refs)
         AND f.deleted_at IS NULL AND f.source_id = ANY($3::text[])
         AND f.id NOT IN (SELECT id FROM tgt)${pagePriv}${pageTrust}
    )`;
}

/** A timestamp as UTC ISO text with microseconds (a cursor compares at full precision). */
const dateText = (column: string) => `to_char(${column} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')`;

function preview(body: string | null, keep: ('private' | 'world')[]): string {
  if (!body) return '';
  const clean = redactFindings(stripFactsFence(stripTakesFence(body), { keepVisibility: keep }), { highEntropy: true }).text
    .replace(/^---[\s\S]*?---\s*/, '')
    .replace(/^\s*#\s[^\n]*\n?/, '');
  const text = clean.replace(/\s+/g, ' ').trim();
  return text.length > PREVIEW_CHARS ? text.slice(0, PREVIEW_CHARS) : text;
}

async function hydrate(engine: BrainEngine, rows: RawRow[], keep: ('private' | 'world')[]): Promise<ReferenceRow[]> {
  if (!rows.length) return [];
  const pages = new Map((await engine.executeRaw<{ id: number; title: string | null; body: string | null; date_source: string;
    trust_tier: string; write_origin: unknown; flagged: boolean; frontmatter: Record<string, unknown> | null }>(
    `SELECT id, title, left(${boundedBodyPrefixSql('pages')}, 4000) AS body,
            CASE WHEN effective_date IS NULL THEN 'updated_at' ELSE COALESCE(effective_date_source, 'effective_date') END AS date_source,
            trust_tier, write_origin, ${activationSuppressedSql('pages', 'pages')} AS flagged, frontmatter
       FROM pages WHERE id = ANY($1::int[])`, [rows.map(r => r.id)])).map(r => [Number(r.id), r]));
  return rows.map(r => {
    const page = pages.get(Number(r.id));
    const fields = page ? trustFields(page.trust_tier, page.write_origin) : { trust_tier: 'unknown' as const, origin: 'unrecorded' };
    return {
      slug: r.slug, source_id: r.source_id, title: page?.title ?? r.slug, type: r.type || null, canonical_type: r.canonical_type,
      date: r.d, date_source: page?.date_source ?? 'updated_at', preview: preview(page?.body ?? null, keep), ...fields,
      ...(page?.flagged === true ? { unconfirmed: true as const } : {}),
      ...(isQuarantined(page?.frontmatter) ? { quarantined: true as const } : {}),
    };
  });
}

export function encodeCursor(row: Pick<ReferenceRow, 'date' | 'source_id' | 'slug'>): string {
  return Buffer.from(JSON.stringify([row.date, row.source_id, row.slug])).toString('base64url');
}

export function decodeCursor(cursor: string): [string | null, string, string] | null {
  try {
    const v = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
    if (Array.isArray(v) && v.length === 3 && (v[0] === null || typeof v[0] === 'string') && typeof v[1] === 'string' && typeof v[2] === 'string') {
      return v as [string | null, string, string];
    }
  } catch { /* invalid */ }
  return null;
}

/** Canonical and stored types observed among the entity's referrers (for the unknown-type error). */
export async function referrerTypes(engine: BrainEngine, scope: ReferrerScope): Promise<string[]> {
  const t = await targetPages(engine, scope);
  const pairs = typeAliasPairs(scope.pack);
  const rows = await engine.executeRaw<{ canonical_type: string; type: string | null }>(
    `${referrersCte(scope)} SELECT DISTINCT canonical_type, type FROM typed`,
    [t.sources, t.slugs, scope.referrerSources, pairs.stored, pairs.canonical]);
  return [...new Set(rows.flatMap(r => [r.canonical_type, ...(r.type ? [r.type] : [])]))].sort();
}

/** One page of referring pages (newest first), optionally of one type. */
export async function readReferrerPage(engine: BrainEngine, scope: ReferrerScope,
  opts: { type?: string; limit: number; cursor?: [string | null, string, string] | null; recordMetadata?: boolean; pageStatus?: boolean }): Promise<{ rows: Array<ReferenceRow & RecordMetadata>; total: number; truncated: boolean; cursor: string | null }> {
  const t = await targetPages(engine, scope);
  const pairs = typeAliasPairs(scope.pack);
  const params: unknown[] = [t.sources, t.slugs, scope.referrerSources, pairs.stored, pairs.canonical, opts.type ?? null];
  const typeFilter = `($6::text IS NULL OR canonical_type = $6 OR type = $6)`;
  const [count] = await engine.executeRaw<{ n: number }>(`${referrersCte(scope)} SELECT count(*)::int AS n FROM typed WHERE ${typeFilter}`, params);
  let after = '';
  if (opts.cursor) {
    params.push(opts.cursor[0], opts.cursor[1], opts.cursor[2]);
    after = ` AND ((d IS NOT DISTINCT FROM $7::timestamptz AND (source_id, slug) > ($8::text, $9::text))
                OR ($7::timestamptz IS NOT NULL AND (d < $7::timestamptz OR d IS NULL)))`;
  }
  params.push(opts.limit + 1);
  const raw = await engine.executeRaw<RawRow>(
    `${referrersCte(scope)}
     SELECT id, slug, source_id, type, canonical_type, ${dateText('d')} AS d FROM typed
      WHERE ${typeFilter}${after}
      ORDER BY d DESC NULLS LAST, source_id, slug LIMIT $${params.length}`, params);
  const truncated = raw.length > opts.limit;
  const page = raw.slice(0, opts.limit);
  const rows: Array<ReferenceRow & RecordMetadata> = await hydrate(engine, page, scope.keepVisibility);
  if (opts.recordMetadata) {
    const meta = await readRecordMetadata(engine, page.map(r => Number(r.id)), scope);
    page.forEach((r, i) => Object.assign(rows[i]!, meta.get(Number(r.id)) ?? {}));
  } else if (opts.pageStatus) {
    const status = await readPageStatusByRef(engine, rows, scope);
    rows.forEach(r => { const value = status.get(`${r.source_id}:${r.slug}`); if (value) r.page_status = value; });
  }
  return { rows, total: Number(count?.n ?? 0), truncated, cursor: truncated ? encodeCursor(rows[rows.length - 1]!) : null };
}

export interface ReferenceGroup {
  canonical_type: string;
  total: number;
  rows: ReferenceRow[];
}

/**
 * The card's groups: every canonical type with its total, newest rows first,
 * at most GROUP_ROW_CAP per group and CARD_ROW_CAP across the card (spread so
 * every group shows rows before any group shows more). Groups are ordered by
 * their newest row.
 */
export async function readReferrerGroups(engine: BrainEngine, scope: ReferrerScope): Promise<{ total: number; groups: ReferenceGroup[] }> {
  const t = await targetPages(engine, scope);
  const pairs = typeAliasPairs(scope.pack);
  // Per-type totals by hash aggregate, then a bounded top-N per type (no full sort of the referrers).
  const raw = await engine.executeRaw<RawRow>(
    `${referrersCte(scope)},
     counts AS (SELECT canonical_type, count(*)::int AS n FROM typed GROUP BY canonical_type)
     SELECT x.id, x.slug, x.source_id, x.type, c.canonical_type, ${dateText('x.d')} AS d, c.n AS gtotal
       FROM counts c CROSS JOIN LATERAL (
         SELECT * FROM typed t WHERE t.canonical_type = c.canonical_type ORDER BY t.d DESC NULLS LAST, t.source_id, t.slug LIMIT $6
       ) x
      ORDER BY c.canonical_type, x.d DESC NULLS LAST, x.source_id, x.slug`,
    [t.sources, t.slugs, scope.referrerSources, pairs.stored, pairs.canonical, GROUP_ROW_CAP]);
  const byType = new Map<string, RawRow[]>();
  for (const r of raw) byType.set(r.canonical_type, [...(byType.get(r.canonical_type) ?? []), r]);
  const order = [...byType.entries()].sort(([a, x], [b, y]) => (y[0]!.d ?? '').localeCompare(x[0]!.d ?? '') || a.localeCompare(b));
  // Water-fill: raise a per-group quota until the card cap is reached.
  const quota = new Map(order.map(([type]) => [type, 0]));
  let used = 0;
  for (let level = 1; level <= GROUP_ROW_CAP && used < CARD_ROW_CAP; level++) {
    for (const [type, rows] of order) {
      if (used >= CARD_ROW_CAP) break;
      if (rows.length >= level) { quota.set(type, level); used++; }
    }
  }
  const kept = order.flatMap(([type, rows]) => rows.slice(0, quota.get(type)));
  const hydrated = await hydrate(engine, kept, scope.keepVisibility);
  const groups: ReferenceGroup[] = order.map(([type, rows]) => ({
    canonical_type: type, total: Number(rows[0]!.gtotal ?? rows.length), rows: hydrated.filter(r => r.canonical_type === type),
  }));
  return { total: groups.reduce((n, g) => n + g.total, 0), groups };
}


// ---------------------------------------------------------------------------
// Records: the referring pages of enrolled record types, for many entities at once
// ---------------------------------------------------------------------------

/** Record rows per target page (all of them when it has this many or fewer). */
export const RECORD_ROWS_PER_TARGET = 5;
/** Record rows per call, allotted to target pages in input order. */
export const RECORD_ROWS_PER_CALL = 24;
/** `amends` / `supersedes` entries kept per record row. */
export const RECORD_LINKS_CAP = 5;
/** A frontmatter `status` longer than this, or not a string, is not shown. */
export const PAGE_STATUS_MAX_CHARS = 40;

/** The metadata a record row adds to a referring page; shared by the record view and its `get_backlinks` continuation. */
export interface RecordMetadata {
  /** The page's own frontmatter `status` (a string up to PAGE_STATUS_MAX_CHARS), only when set. */
  page_status?: string;
  /** Records this page amends (an `amends` link), each with the terms-start date its link records (`since`), when recorded. */
  amends?: Array<{ slug: string; since?: string }>;
  /** Records this page replaces as a whole (a `supersedes` link). */
  supersedes?: Array<{ slug: string }>;
}

/** One record: the referring page, its own date labeled as a page date with its source, and its record metadata. */
export interface RecordRow extends RecordMetadata {
  slug: string;
  title: string;
  /** The page's own date (when it was written or dated), not when its terms take effect. */
  date: string | null;
  date_source: string;
}

export interface RecordGroupRaw {
  canonical_type: string;
  total: number;
  rows: RecordRow[];
  /** The cursor that continues this type after the last shown row (null when no row is shown). */
  cursor: string | null;
  /** Why rows < total: the per-page cap, or the per-call cap (an earlier entity in the call took the rows). */
  capped?: 'per_page' | 'per_call';
}

export interface TargetReferences {
  /** Distinct referring pages, every type (the card's `referenced_by_count`). */
  referenced_by_count: number;
  /** Record groups, one per enrolled type with at least one record, newest group first. Absent when nothing is enrolled. */
  records?: RecordGroupRaw[];
}

export interface RecordTarget { slug: string; sourceId: string }

/** Enrolled types as pack-canonical types (a stored alias maps to its type), in enrollment order. */
export function canonicalRecordTypes(types: string[], pack: PackTypes | null): string[] {
  const pairs = typeAliasPairs(pack);
  return [...new Set(types.map(t => pairs.canonical[pairs.stored.indexOf(t)] ?? t))];
}

/**
 * The batched variant of `referrersCte`: the same membership, filters and
 * date, with a target index `i` on every row so many entities are read in
 * one statement. Params $1..$3 are the targets (sources, slugs, indexes);
 * $4 referrer sources; $5/$6 the stored type aliases and their canonical types.
 */
function referrersCteBatched(scope: Pick<ReferrerScope, 'excludePrivate' | 'eligibility' | 'includeQuarantined'>): string {
  const { targetPriv, linkPriv, pagePriv, pageTrust } = referrerFilters(scope);
  return `WITH tgt AS MATERIALIZED (
      SELECT k.i, t.id FROM pages t JOIN unnest($1::text[], $2::text[], $3::int[]) AS k(s, g, i) ON t.source_id = k.s AND t.slug = k.g
       WHERE t.deleted_at IS NULL${targetPriv}
    ),
    refs AS MATERIALIZED (
      SELECT DISTINCT tg.i, l.from_page_id AS id FROM links l JOIN tgt tg ON l.to_page_id = tg.id WHERE true${linkPriv}
    ),
    typed AS MATERIALIZED (
      SELECT r.i, f.id, f.slug, f.source_id, f.type,
             COALESCE(f.effective_date, f.updated_at) AS d,
             COALESCE(($6::text[])[array_position($5::text[], f.type)], NULLIF(f.type, ''), 'untyped') AS canonical_type
        FROM refs r JOIN pages f ON f.id = r.id
       WHERE f.deleted_at IS NULL AND f.source_id = ANY($4::text[])
         AND NOT EXISTS (SELECT 1 FROM tgt x WHERE x.i = r.i AND x.id = f.id)${pagePriv}${pageTrust}
    )`;
}

/** A frontmatter `status` as `page_status`: a string, trimmed, at most PAGE_STATUS_MAX_CHARS; anything else is not shown. */
export function pageStatusValue(raw: unknown): string | undefined {
  const value = typeof raw === 'string' ? raw.trim() : '';
  return value && value.length <= PAGE_STATUS_MAX_CHARS ? value : undefined;
}

/**
 * F3: `page_status` of live pages by `(source_id, slug)`, keyed
 * `source_id:slug`, one statement. A private page (for an untrusted caller)
 * or a quarantined page (unless the caller may read quarantined pages) gets
 * none.
 */
export async function readPageStatusByRef(engine: BrainEngine, refs: Array<{ source_id: string; slug: string }>,
  scope: { excludePrivate: boolean; includeQuarantined?: boolean }): Promise<Map<string, string>> {
  const out = new Map<string, string>();
  if (!refs.length) return out;
  const priv = scope.excludePrivate ? ` AND ${privatePagesFilterForCaller('p')}` : '';
  const quarantine = scope.includeQuarantined ? '' : ` AND ${quarantineFilterFragment('p')}`;
  const rows = await engine.executeRaw<{ source_id: string; slug: string; status: unknown }>(
    `SELECT p.source_id, p.slug, p.frontmatter->'status' AS status
       FROM pages p JOIN unnest($1::text[], $2::text[]) AS u(s, g) ON p.source_id = u.s AND p.slug = u.g
      WHERE p.deleted_at IS NULL AND p.frontmatter ? 'status'${priv}${quarantine}`,
    [refs.map(r => r.source_id), refs.map(r => r.slug)]);
  for (const row of rows) {
    const value = pageStatusValue(row.status);
    if (value) out.set(`${row.source_id}:${row.slug}`, value);
  }
  return out;
}

/**
 * The record metadata of referring pages: `page_status` from frontmatter,
 * `amends` and `supersedes` links to pages the caller can see (targets in the
 * caller's referrer sources, link origins under the same privacy rule as the
 * referrer read), and each amends link's
 * `since` from the temporal relationship state for the caller's scope
 * (`world` for untrusted callers, so a date whose only evidence is private
 * stays hidden).
 */
export async function readRecordMetadata(engine: BrainEngine, pageIds: number[],
  scope: Pick<ReferrerScope, 'referrerSources' | 'excludePrivate' | 'includeQuarantined'>): Promise<Map<number, RecordMetadata>> {
  const out = new Map<number, RecordMetadata>();
  if (!pageIds.length) return out;
  const linkPriv = scope.excludePrivate ? ` AND ${privateLinkOriginFilterForCaller('l')}` : '';
  const targetPriv = scope.excludePrivate ? ` AND ${privatePagesFilterForCaller('t')}` : '';
  const targetQuarantine = scope.includeQuarantined ? '' : ` AND ${quarantineFilterFragment('t')}`;
  const [status, links] = await Promise.all([
    engine.executeRaw<{ id: number; status: unknown }>(`SELECT id, frontmatter->'status' AS status FROM pages WHERE id = ANY($1::int[])`, [pageIds]),
    engine.executeRaw<{ from_id: number; slug: string; link_type: string; since: string | null }>(
      `SELECT DISTINCT l.from_page_id AS from_id, t.slug, l.link_type,
              CASE WHEN l.link_type = 'amends' THEN to_char(r.first_start, 'YYYY-MM-DD') END AS since
         FROM links l JOIN pages t ON t.id = l.to_page_id AND t.deleted_at IS NULL AND t.source_id = ANY($3::text[])${targetPriv}${targetQuarantine}
         LEFT JOIN link_relationships r ON r.from_page_id = l.from_page_id AND r.to_page_id = l.to_page_id
              AND r.link_type = l.link_type AND r.scope = $2
        WHERE l.from_page_id = ANY($1::int[]) AND l.link_type IN ('amends', 'supersedes')${linkPriv}
        ORDER BY l.from_page_id, t.slug`, [pageIds, scope.excludePrivate ? 'world' : 'all', scope.referrerSources]).catch(() => []),
  ]);
  for (const row of status) {
    const value = pageStatusValue(row.status);
    if (value) out.set(Number(row.id), { page_status: value });
  }
  for (const link of links) {
    const id = Number(link.from_id);
    const meta = out.get(id) ?? {};
    if (link.link_type === 'amends') {
      const list = meta.amends ?? [];
      const existing = list.find(a => a.slug === link.slug);
      if (existing) { if (!existing.since && link.since) existing.since = link.since; }
      else if (list.length < RECORD_LINKS_CAP) list.push({ slug: link.slug, ...(link.since ? { since: link.since } : {}) });
      meta.amends = list;
    } else {
      const list = meta.supersedes ?? [];
      if (!list.some(s => s.slug === link.slug) && list.length < RECORD_LINKS_CAP) list.push({ slug: link.slug });
      meta.supersedes = list;
    }
    out.set(id, meta);
  }
  return out;
}

interface RecordCandidate { i: number; id: number; slug: string; source_id: string; canonical_type: string; d: string | null }

/**
 * `referenced_by_count` and the enrolled-type records of many target pages in
 * one batched read. Each target expands like the single-target read (its
 * identity co-members under `entity_identity.union`), and its rows, totals
 * and cursors equal `readReferrerPage(…, { type })` for that target: records
 * are filtered by type inside the membership query, before ORDER BY, LIMIT
 * and hydration. Rows per target are capped at RECORD_ROWS_PER_TARGET, newest
 * page date first across the target's record types; the call cap is applied
 * after, in target order, before hydration. Targets repeating a page reuse
 * its result without taking more of the call cap.
 */
export async function readTargetReferences(engine: BrainEngine,
  scope: Omit<ReferrerScope, 'slug' | 'sourceId'>, targets: RecordTarget[],
  opts: { recordTypes: string[]; perTarget?: number; perCall?: number; isExpired?: () => boolean }): Promise<TargetReferences[]> {
  const keyOf = (t: RecordTarget) => `${t.sourceId}\0${t.slug}`;
  const unique: RecordTarget[] = [];
  const index = new Map<string, number>();
  for (const t of targets) if (!index.has(keyOf(t))) { index.set(keyOf(t), unique.length); unique.push(t); }
  if (!unique.length) return [];
  const union = await isIdentityUnionEnabled(engine);
  const expanded = await Promise.all(unique.map(t => targetPages(engine, { ...scope, slug: t.slug, sourceId: t.sourceId }, union)));
  const sources: string[] = [];
  const slugs: string[] = [];
  const idx: number[] = [];
  expanded.forEach((e, i) => e.slugs.forEach((slug, j) => { sources.push(e.sources[j]!); slugs.push(slug); idx.push(i); }));
  const pairs = typeAliasPairs(scope.pack);
  const recordTypes = canonicalRecordTypes(opts.recordTypes, scope.pack);
  const base = [sources, slugs, idx, scope.referrerSources, pairs.stored, pairs.canonical];
  const expired = () => opts.isExpired?.() === true;
  const counts = await engine.executeRaw<{ i: number; canonical_type: string | null; n: number }>(
    `${referrersCteBatched(scope)}
     SELECT i, NULL::text AS canonical_type, count(*)::int AS n FROM typed GROUP BY i
     UNION ALL
     SELECT i, canonical_type, count(*)::int AS n FROM typed WHERE canonical_type = ANY($7::text[]) GROUP BY i, canonical_type`,
    [...base, recordTypes]);
  if (expired()) throw new ReferencesDeadlineError();
  const perTarget = opts.perTarget ?? RECORD_ROWS_PER_TARGET;
  const candidates = recordTypes.length ? await engine.executeRaw<RecordCandidate>(
    `${referrersCteBatched(scope)}
     SELECT x.i, x.id, x.slug, x.source_id, x.canonical_type, ${dateText('x.d')} AS d
       FROM (SELECT DISTINCT i FROM tgt) q CROSS JOIN LATERAL (
         SELECT * FROM typed t WHERE t.i = q.i AND t.canonical_type = ANY($7::text[])
          ORDER BY t.d DESC NULLS LAST, t.source_id, t.slug LIMIT $8
       ) x
      ORDER BY x.i, x.d DESC NULLS LAST, x.source_id, x.slug`, [...base, recordTypes, perTarget]) : [];
  if (expired()) throw new ReferencesDeadlineError();
  let left = opts.perCall ?? RECORD_ROWS_PER_CALL;
  const kept: RecordCandidate[] = [];
  const callCut = new Set<number>();
  for (let i = 0; i < unique.length; i++) {
    const mine = candidates.filter(c => Number(c.i) === i);
    const rows = mine.slice(0, Math.max(0, left));
    if (rows.length < mine.length) callCut.add(i);
    left -= rows.length;
    kept.push(...rows);
  }
  const hydrated = await hydrateRecords(engine, kept, scope);
  if (expired()) throw new ReferencesDeadlineError();
  const results = unique.map((_, i): TargetReferences => {
    const total = counts.find(c => Number(c.i) === i && c.canonical_type === null)?.n ?? 0;
    if (!recordTypes.length) return { referenced_by_count: Number(total) };
    const typeTotals = counts.filter(c => Number(c.i) === i && c.canonical_type !== null);
    const mine = kept.map((k, j) => ({ k, row: hydrated[j]! })).filter(x => Number(x.k.i) === i);
    const groups = typeTotals.map((c): RecordGroupRaw & { newest: string } => {
      const shown = mine.filter(x => x.k.canonical_type === c.canonical_type);
      const last = shown[shown.length - 1]?.k;
      const capped = shown.length >= Number(c.n) ? undefined : callCut.has(i) ? 'per_call' as const : 'per_page' as const;
      return { canonical_type: c.canonical_type!, total: Number(c.n), rows: shown.map(x => x.row),
        cursor: last ? encodeCursor({ date: last.d, source_id: last.source_id, slug: last.slug }) : null,
        ...(capped ? { capped } : {}), newest: shown[0]?.k.d ?? '' };
    }).sort((a, b) => b.newest.localeCompare(a.newest) || recordTypes.indexOf(a.canonical_type) - recordTypes.indexOf(b.canonical_type))
      .map(({ newest: _n, ...g }) => g);
    return { referenced_by_count: Number(total), records: groups };
  });
  return targets.map(t => results[index.get(keyOf(t))!]!);
}

async function hydrateRecords(engine: BrainEngine, rows: RecordCandidate[],
  scope: Pick<ReferrerScope, 'referrerSources' | 'excludePrivate' | 'includeQuarantined'>): Promise<RecordRow[]> {
  if (!rows.length) return [];
  const ids = [...new Set(rows.map(r => Number(r.id)))];
  const [pages, meta] = await Promise.all([
    engine.executeRaw<{ id: number; title: string | null; date_source: string }>(
      `SELECT id, title, CASE WHEN effective_date IS NULL THEN 'updated_at' ELSE COALESCE(effective_date_source, 'effective_date') END AS date_source
         FROM pages WHERE id = ANY($1::int[])`, [ids]),
    readRecordMetadata(engine, ids, scope),
  ]);
  const byId = new Map(pages.map(p => [Number(p.id), p]));
  return rows.map(r => {
    const page = byId.get(Number(r.id));
    return { slug: r.slug, title: page?.title ?? r.slug, date: r.d, date_source: page?.date_source ?? 'updated_at', ...(meta.get(Number(r.id)) ?? {}) };
  });
}

/** The records/count phase ran past its deadline; callers show the unavailable marker. */
export class ReferencesDeadlineError extends Error {
  constructor() { super('references deadline'); this.name = 'ReferencesDeadlineError'; }
}