Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.2 KB
Raw Blame
/**
 * Storage for wanted pages (see src/core/wanted-links.ts). Kept free of the
 * link-extraction import graph because derived-links.ts, which every engine
 * imports, calls it inside the derived-link replacement transaction.
 */
import type { BrainEngine } from './engine.ts';
import { executeRawJsonb } from './sql-query.ts';
import { sanitizeForJsonb } from './batch-rows.ts';
import { validateSlug } from './utils.ts';

export type WantedProducer = 'body' | 'frontmatter';

export interface WantedLinkInput {
  producer: WantedProducer;
  ref_kind: 'slug' | 'name';
  target_source_id: string;
  target_ref: string;
  link_type: string;
  context: string;
}

export interface WantedLinksReplacement {
  rows: WantedLinkInput[];
  /** Producers whose rows this replacement owns; others stay untouched. */
  producers: readonly WantedProducer[];
  /**
   * wantedReadAt() taken before the extraction's first resolution read. Every
   * `name` row is stamped with it, so a page written after that read wakes the
   * origin and one the read already saw does not. Required when rows has a name.
   */
  readAt?: string;
}

/** The database clock as a UTC ISO string with microseconds, independent of the session's DateStyle. */
export async function wantedReadAt(engine: Pick<BrainEngine, 'executeRaw'>): Promise<string> {
  const [row] = await engine.executeRaw<{ at: string }>(
    `SELECT to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS at`);
  return row!.at;
}

/**
 * #6228/#6225: the rows whose target can become a page. A slug target must
 * pass validateSlug (a regex literal on a code page read as a wikilink does
 * not) and every target must be in a registered source (a `[[memory:123]]`
 * citation names none). Any other row names no possible page; locking it
 * would throw and abort the caller's whole extraction transaction. Foreign
 * sources are read `FOR SHARE`, so one cannot disappear between this check
 * and the page-guard lock; the origin's own source needs no read.
 */
export async function possibleWantedRows<T extends Pick<WantedLinkInput, 'ref_kind' | 'target_source_id' | 'target_ref'>>(
  tx: Pick<BrainEngine, 'executeRaw'>, originSourceId: string, rows: readonly T[],
): Promise<T[]> {
  const shaped = rows.filter(row => row.ref_kind !== 'slug' || pageSlugShaped(row.target_ref));
  const foreign = [...new Set(shaped.map(row => row.target_source_id).filter(id => id !== originSourceId))];
  if (!foreign.length) return shaped;
  const registered = new Set((await tx.executeRaw<{ id: string }>(
    'SELECT id FROM sources WHERE id = ANY($1::text[]) ORDER BY id FOR SHARE', [foreign])).map(row => row.id));
  return shaped.filter(row => row.target_source_id === originSourceId || registered.has(row.target_source_id));
}

function pageSlugShaped(target: string): boolean {
  try { validateSlug(target); return true; } catch { return false; } // a refused slug is the answer, not an error
}

/**
 * Replace one origin's wanted rows inside the caller's transaction (the same
 * one that replaces its derived links), keeping only possibleWantedRows. Slug targets are locked first, so a
 * concurrent writer creating the target either committed before this check
 * (and the row is stamped `-infinity`, leaving the origin stale) or commits
 * after it with an `updated_at` past `checked_at`. A bare-name reference may
 * legitimately stay unresolved while a page has its exact slug or basename
 * (basename resolution is opt-in, and a directory hint can skip a top-level
 * slug), so it is stamped with the caller's read time instead: a page written
 * after that read wakes the origin once, and the next extraction settles it.
 */
export async function replaceWantedLinks(
  tx: Pick<BrainEngine, 'executeRaw' | 'lockPageKeys'>,
  origin: { pageId: number; sourceId: string },
  replacement: WantedLinksReplacement,
): Promise<number> {
  if (!replacement.producers.length) return 0;
  if (!replacement.readAt && replacement.rows.some(row => row.ref_kind === 'name')) {
    throw new Error('replaceWantedLinks: name rows need the readAt taken before the extraction read');
  }
  const rows = (await possibleWantedRows(tx, origin.sourceId,
    replacement.rows.filter(row => replacement.producers.includes(row.producer))))
    .map(row => ({ ...row, context: sanitizeForJsonb(row.context) }));
  const slugTargets = rows.filter(row => row.ref_kind === 'slug')
    .map(row => ({ sourceId: row.target_source_id, slug: row.target_ref }));
  if (slugTargets.length) await tx.lockPageKeys(slugTargets);
  await executeRawJsonb(tx, `DELETE FROM wanted_links w
      WHERE w.origin_page_id = $1
        AND w.producer IN (SELECT jsonb_array_elements_text(($2::jsonb)->'producers'))
        AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(($2::jsonb)->'rows')
          AS v(producer text, ref_kind text, target_source_id text, target_ref text)
          WHERE v.producer = w.producer AND v.ref_kind = w.ref_kind
            AND v.target_source_id = w.target_source_id AND v.target_ref = w.target_ref)`,
  [origin.pageId], [{ rows, producers: replacement.producers }]);
  if (!rows.length) return 0;
  const written = await executeRawJsonb<{ id: number }>(tx, `INSERT INTO wanted_links
      (origin_page_id, source_id, producer, ref_kind, target_source_id, target_ref, link_type, context, checked_at)
    SELECT $1, $2, v.producer, v.ref_kind, v.target_source_id, v.target_ref, v.link_type, v.context,
      CASE WHEN v.ref_kind = 'slug' AND EXISTS (SELECT 1 FROM pages t WHERE t.source_id = v.target_source_id
        AND t.deleted_at IS NULL AND t.slug = v.target_ref)
      THEN '-infinity'::timestamptz WHEN v.ref_kind = 'name' THEN $3::timestamptz ELSE now() END
    FROM jsonb_to_recordset(($4::jsonb)->'rows')
      AS v(producer text, ref_kind text, target_source_id text, target_ref text, link_type text, context text)
    ON CONFLICT ON CONSTRAINT wanted_links_reference_unique DO UPDATE
      SET link_type = EXCLUDED.link_type, context = EXCLUDED.context, checked_at = EXCLUDED.checked_at
    RETURNING id`, [origin.pageId, origin.sourceId, replacement.readAt ?? null], [{ rows }]);
  return written.length;
}

export interface WantedPageRow {
  target: string;
  target_source_id: string;
  ref_kind: 'slug' | 'name';
  referenced_by: number;
  sample_origins: Array<{ slug: string; source_id: string }>;
  link_types: string[];
  /** Live pages whose basename equals a bare-name target (link them by slug). */
  existing_matches: Array<{ slug: string; source_id: string }>;
  first_seen: string;
  last_checked: string;
}

/**
 * Missing link targets, most-referenced first. Scope applies to the origin
 * pages (the caller's readable sources) and, when `excludePrivate`, private
 * origins are removed BEFORE aggregation so a private page never reveals a
 * target name or contributes to a count. Soft-deleted origins never count.
 */
export async function listWantedPages(engine: Pick<BrainEngine, 'executeRaw'>, opts: {
  sourceId?: string; sourceIds?: string[]; excludePrivate?: boolean; privateFilter?: (alias: string) => string;
  limit: number; offset: number;
}): Promise<{ total: number; rows: WantedPageRow[] }> {
  const params: unknown[] = [];
  const scope = opts.sourceIds?.length
    ? (params.push(opts.sourceIds), ` AND o.source_id = ANY($${params.length}::text[])`)
    : opts.sourceId ? (params.push(opts.sourceId), ` AND o.source_id = $${params.length}`) : '';
  const originPrivacy = opts.excludePrivate && opts.privateFilter ? ` AND ${opts.privateFilter('o')}` : '';
  const matchPrivacy = opts.excludePrivate && opts.privateFilter ? ` AND ${opts.privateFilter('m')}` : '';
  const base = `WITH wanted AS (
      SELECT w.target_source_id, w.target_ref, w.ref_kind, w.link_type, w.first_seen_at, w.checked_at,
        o.slug AS origin_slug, o.source_id AS origin_source_id
      FROM wanted_links w JOIN pages o ON o.id = w.origin_page_id AND o.deleted_at IS NULL
      WHERE NOT EXISTS (SELECT 1 FROM pages t WHERE t.source_id = w.target_source_id AND t.slug = w.target_ref AND t.deleted_at IS NULL)
        ${scope}${originPrivacy}
    ), grouped AS (
      SELECT target_source_id, target_ref, min(ref_kind) AS ref_kind,
        count(DISTINCT (origin_source_id, origin_slug))::int AS referenced_by,
        jsonb_agg(jsonb_build_object('slug', origin_slug, 'source_id', origin_source_id) ORDER BY origin_slug) AS origins,
        coalesce(jsonb_agg(DISTINCT link_type) FILTER (WHERE link_type <> ''), '[]'::jsonb) AS link_types,
        min(first_seen_at) AS first_seen, max(checked_at) AS last_checked
      FROM wanted GROUP BY target_source_id, target_ref
    )`;
  const [{ total }] = await engine.executeRaw<{ total: number }>(`${base} SELECT count(*)::int AS total FROM grouped`, params);
  params.push(opts.limit, opts.offset);
  const rows = await engine.executeRaw<{ target_source_id: string; target_ref: string; ref_kind: 'slug' | 'name'; referenced_by: number;
    origins: Array<{ slug: string; source_id: string }>; link_types: string[]; matches: Array<{ slug: string; source_id: string }> | null;
    first_seen: Date | string; last_checked: Date | string }>(`${base}
    SELECT g.*, (SELECT jsonb_agg(jsonb_build_object('slug', m.slug, 'source_id', m.source_id))
        FROM (SELECT m.slug, m.source_id FROM pages m WHERE g.ref_kind = 'name' AND m.source_id = g.target_source_id
          AND regexp_replace(m.slug, '^.*/', '') = g.target_ref AND m.deleted_at IS NULL${matchPrivacy}
          ORDER BY m.slug LIMIT 3) m) AS matches
    FROM grouped g ORDER BY g.referenced_by DESC, g.target_source_id, g.target_ref
    LIMIT $${params.length - 1} OFFSET $${params.length}`, params);
  const iso = (value: Date | string) => (value instanceof Date ? value : new Date(value)).toISOString();
  return { total: Number(total ?? 0), rows: rows.map(row => {
    const seen = new Set<string>();
    const origins = (row.origins ?? []).filter(o => !seen.has(`${o.source_id}\0${o.slug}`) && seen.add(`${o.source_id}\0${o.slug}`));
    return { target: row.target_ref, target_source_id: row.target_source_id, ref_kind: row.ref_kind,
      referenced_by: Number(row.referenced_by), sample_origins: origins.slice(0, 3), link_types: row.link_types ?? [],
      existing_matches: row.matches ?? [], first_seen: iso(row.first_seen),
      last_checked: Number.isFinite(new Date(row.last_checked).getTime()) ? iso(row.last_checked) : 'pending' };
  }) };
}