Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
25.3 KB
Raw Blame
import type { BrainEngine, LinkBatchInput } from './engine.ts';
import { extractPageLinks, unwrapWikilink, resolvedLinkCandidate,
  isCrossSourceLinksEnabled, type LinkCandidate, type LinkExtractionPack, type PageLinksResult, type SlugResolver } from './link-extraction.ts';
import { fetchSource, isSourceFederated } from './sources-load.ts';
import { parseMarkdown } from './markdown.ts';
import { isValidSourceId } from './source-id.ts';
import { buildSourceLocalReferenceIndex } from './source-local-reference-index.ts';
import { collectWantedLinks, isWantedPagesEnabled, wantedReadAt } from './wanted-links.ts';
import { readLineGrammarSettings } from './line-grammar.ts';
import { DerivedLinkSettingsChangedError } from './derived-links.ts';
import { isSegmentedPage } from './page-body/page-shape.ts';
import { extractSegmentedPageLinks } from './page-body/segment-links.ts';
import type { TemporalBody } from './link-temporal-apply.ts';

export interface LinkPageMetadata {
  slug: string;
  source_id: string;
  type: string;
  title: string;
  aliases?: unknown;
  knowledge_revision: string;
}

export async function loadLinkPageMetadata(engine: Pick<BrainEngine, 'executeRaw'>, sourceId?: string): Promise<LinkPageMetadata[]> {
  return engine.executeRaw<LinkPageMetadata>(`SELECT slug, source_id, type, title, frontmatter->'aliases' AS aliases, knowledge_revision FROM pages
    WHERE deleted_at IS NULL${sourceId ? ' AND source_id=$1' : ''} ORDER BY source_id, slug`, sourceId ? [sourceId] : []);
}

export function makeIndexedLinkResolver(pages: readonly LinkPageMetadata[], sourceId: string): SlugResolver {
  const index = buildSourceLocalReferenceIndex(pages.filter(page => page.source_id === sourceId));
  return {
    async resolveAttendance(name, dirHint) { return this.resolve(name, dirHint); },
    async resolve(name, dirHint) {
      if (!name) return null;
      let value = name.trim();
      const colon = value.indexOf(':');
      if (colon !== -1 && isValidSourceId(value.slice(0, colon))) {
        if (value.slice(0, colon) !== sourceId) return null;
        value = value.slice(colon + 1);
      }
      const matches = index.resolveMatches(value, dirHint);
      return matches.length === 1 ? matches[0] : null;
    },
    async resolveBasenameMatches(name) { return index.basenameMatches(name); },
  };
}

export interface SourceLinkReconciliationResult {
  ok: boolean;
  complete: boolean;
  pagesProcessed: number;
  linksCreated: number;
  linksRemoved: number;
  nextAfterSlug?: string;
  unresolved: Array<{ originSlug: string; field?: string; target: string; reason: 'missing_target' | 'cross_source' | 'target_type_mismatch' }>;
  failures: Array<{ originSlug?: string; code: string }>;
}

export async function reconcileSourceLinks(
  engine: BrainEngine,
  sourceId: string,
  opts: { pack: LinkExtractionPack; afterSlug?: string; limit?: number; globalBasename?: boolean;
    expectedSourceIncarnation?: string },
): Promise<SourceLinkReconciliationResult> {
  if (!isValidSourceId(sourceId)) throw new TypeError('An exact valid source ID is required for reconciliation');
  const limit = opts.limit ?? Infinity;
  if (opts.limit !== undefined && (!Number.isInteger(limit) || limit < 1 || limit > 1000)) throw new TypeError('Reconciliation limit must be between 1 and 1000');
  const result: SourceLinkReconciliationResult = { ok: false, complete: false, pagesProcessed: 0,
    linksCreated: 0, linksRemoved: 0, nextAfterSlug: opts.afterSlug, unresolved: [], failures: [] };
  let originSlug: string | undefined;
  try {
    const sources = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation FROM sources WHERE id=$1', [sourceId]);
    if (!sources.length || (opts.expectedSourceIncarnation && sources[0].incarnation !== opts.expectedSourceIncarnation)) {
      result.failures.push({ code: 'source_identity_changed' });
      return result;
    }
    const sourceIncarnation = sources[0].incarnation;
    // #4680: every source's live pages, so the source policy can resolve a target into another
    // source (names still resolve within the origin's source: makeIndexedLinkResolver filters).
    const readAt = await wantedReadAt(engine);
    const pages = await loadLinkPageMetadata(engine);
    const metadata = new Map(pages.map(page => [`${page.source_id}\0${page.slug}`, page]));
    const endpoints = indexLinkSources(pages);
    const policy = await loadLinkSourcePolicy(engine, sourceId);
    const resolver = makeIndexedLinkResolver(pages, sourceId);
    const wantedEnabled = await isWantedPagesEnabled(engine);
    const grammar = await readLineGrammarSettings(engine);
    const lineGrammar = { enabled: grammar.enabled, allowUndeclaredTypes: grammar.allowUndeclaredTypes };
    const remaining = pages.filter(page => page.source_id === sourceId && (!opts.afterSlug || page.slug > opts.afterSlug))
      .sort((a, b) => a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0);
    for (const own of remaining.slice(0, limit)) {
      originSlug = own.slug;
      const snapshot = await engine.readPageSnapshot(originSlug, { sourceId });
      if (!snapshot || snapshot.revision !== own.knowledge_revision || snapshot.sourceIncarnation !== sourceIncarnation) {
        result.failures.push({ originSlug, code: 'revision_conflict' });
        return result;
      }
      const page = snapshot.page;
      const sourcesOf = makeLinkSourceResolver(page.slug, sourceId, endpoints, policy);
      const linkOpts = { pack: opts.pack, globalBasename: opts.globalBasename, lineGrammar,
        targetType: (slug: string, source?: string) => {
          const resolved = sourcesOf.resolve({ targetSlug: slug, targetSourceId: source, linkType: '', context: '' });
          return resolved.ok ? metadata.get(`${resolved.toSourceId}\0${slug}`)?.type : undefined;
        } };
      // A segmented body is read block by block, and its temporal inputs with it (the publication never reads it).
      const extracted: PageLinksResult & { temporalBody?: TemporalBody } = isSegmentedPage(page)
        ? await extractSegmentedPageLinks(engine, { id: Number(page.id), slug: page.slug, source_id: sourceId, type: page.type, frontmatter: page.frontmatter },
          resolver, { ...linkOpts, temporal: true })
        : await extractPageLinks(page.slug, `${page.compiled_truth}\n${page.timeline}`, page.frontmatter, page.type, resolver, linkOpts);
      for (const ref of extracted.unresolved) {
        const target = unwrapWikilink(ref.name);
        const qualifier = target.split(':')[0];
        const foreign = target.includes(':') && isValidSourceId(qualifier) && qualifier !== sourceId;
        result.unresolved.push({ originSlug: page.slug, field: ref.field, target: ref.name,
          reason: ref.reason ?? (foreign ? 'cross_source' : 'missing_target') });
      }
      if (!extracted.attendanceComplete) {
        result.failures.push({ originSlug, code: 'attendance_resolution_incomplete' });
        return result;
      }
      const rows: LinkBatchInput[] = [];
      for (const candidate of extracted.candidates) {
        const admitted = sourcesOf.admit(candidate);
        if (admitted.ok) { rows.push(admitted.row); continue; }
        const crossSource = admitted.reason === 'cross_source';
        result.unresolved.push({ originSlug, reason: crossSource ? 'cross_source' : 'missing_target',
          target: crossSource && candidate.targetSourceId ? `${candidate.targetSourceId}:${candidate.targetSlug}` : candidate.targetSlug });
      }
      const wanted = wantedEnabled ? collectWantedLinks({ candidates: extracted.candidates, frontmatterUnresolved: extracted.unresolved,
        originSourceId: sourceId, crossSourceAllowed: sourcesOf.crossSourceAllowed, resolve: sourcesOf.resolve }) : [];
      let written: { created: number; removed: number };
      try {
        written = await engine.replaceDerivedLinks({ slug: page.slug, sourceId,
          expectedRevision: snapshot.revision, sourceIncarnation }, rows, { lineGrammar: grammar,
          wanted: { producers: ['body', 'frontmatter'], rows: wanted, readAt }, expectedEndpoints: capturedLinkEndpoints(rows, metadata),
          ...(extracted.temporalBody ? { temporalBody: extracted.temporalBody } : {}) });
      } catch (error) {
        if (!(error instanceof DerivedLinkSettingsChangedError)) throw error;
        result.failures.push({ originSlug, code: 'revision_conflict' });
        return result;
      }
      result.pagesProcessed++;
      result.linksCreated += written.created;
      result.linksRemoved += written.removed;
      result.nextAfterSlug = page.slug;
    }
    result.ok = true;
    result.complete = remaining.length <= limit;
    return result;
  } catch (error) {
    const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
      ? error.code : 'graph_write_failed';
    result.failures.push({ originSlug, code });
    return result;
  }
}

export async function replaceFileLinks(engine: BrainEngine, slug: string, sourceId: string,
  links: LinkBatchInput[], includeFrontmatter: boolean,
  snapshot: NonNullable<Awaited<ReturnType<BrainEngine['readPageSnapshot']>>>,
  ownership: Awaited<ReturnType<typeof fileLinkOwnership>>, content: string,
  opts: { pack: LinkExtractionPack | null; globalBasename: boolean }): Promise<number | null | undefined> {
  const { metadata, resolver, resolve } = ownership;
  const parsed = parseMarkdown(content, `${slug}.md`, { activePack: opts.pack?.page_types ? { page_types: opts.pack.page_types } : undefined });
  const extracted = await extractPageLinks(slug, content, parsed.frontmatter, snapshot.page.type, resolver,
    { ...opts, skipFrontmatter: !includeFrontmatter, targetType: (targetSlug, targetSourceId) => {
      const result = resolve(slug, { targetSlug, targetSourceId, linkType: '', context: '' });
      return result.ok ? metadata.get(`${result.toSourceId}\0${targetSlug}`)?.type : undefined;
    } });
  if (!extracted.attendanceComplete) return null;
  const attendance = extracted.candidates.filter(candidate => candidate.canonicalAttendance
    || (candidate.linkType === 'attended' && candidate.fromSlug && candidate.targetSlug === slug));
  if (!ownership.origins.has(slug) && !attendance.length
    && !links.some(link => link.link_type === 'attended' && link.to_slug === slug && link.origin_slug === slug)) return undefined;
  const rows: LinkBatchInput[] = links.filter(link => !(link.link_type === 'attended' && link.to_slug === slug && link.origin_slug === slug)
    && metadata.has(`${sourceId}\0${link.from_slug}`)
    && metadata.has(`${sourceId}\0${link.to_slug}`)).map(link => ({ ...link,
    from_source_id: sourceId, to_source_id: sourceId, origin_source_id: sourceId }));
  for (const candidate of attendance) {
    const resolved = resolve(slug, candidate);
    if (!resolved.ok) return null;
    rows.push(resolvedLinkCandidate(candidate, slug, sourceId, resolved));
  }
  const result = await engine.replaceDerivedLinks({ slug, sourceId, expectedRevision: snapshot.revision,
    sourceIncarnation: snapshot.sourceIncarnation }, rows, { includeFrontmatter, preserveExisting: true,
    includeLegacyNullProducer: false,
    expectedEndpoints: capturedLinkEndpoints(rows, metadata) });
  return result.created;
}

/**
 * Replace one file page's own markdown-derived links with what its current
 * text produces, the put_page contract: an edge the text no longer supports
 * is removed, other producers' edges stay. Returns the number of new edges,
 * or null when the page row is missing. Links whose endpoints are not live
 * pages in the source are dropped, as the add-only path's failed inserts were.
 */
export async function replacePageFileLinks(engine: BrainEngine, slug: string, sourceId: string,
  links: LinkBatchInput[], includeFrontmatter: boolean,
  ownership: Awaited<ReturnType<typeof fileLinkOwnership>>): Promise<number | null> {
  const snapshot = await engine.readPageSnapshot(slug, { sourceId });
  if (!snapshot) return null;
  const rows = links.filter(link => ownership.metadata.has(`${sourceId}\0${link.from_slug}`)
    && ownership.metadata.has(`${sourceId}\0${link.to_slug}`)).map(link => ({ ...link,
    from_source_id: sourceId, to_source_id: sourceId, origin_source_id: sourceId }));
  const result = await engine.replaceDerivedLinks({ slug, sourceId, expectedRevision: snapshot.revision,
    sourceIncarnation: snapshot.sourceIncarnation }, rows, { includeFrontmatter, preserveExisting: true,
    includeLegacyNullProducer: false });
  return result.created;
}

/**
 * Page metadata, resolver and attendance origins for replacing file links.
 * With `slugs`, only those pages of the source are loaded (a sync hook passes
 * its changed pages plus their link endpoints) — enough for
 * `replacePageFileLinks`; the resolver then only knows those pages, so
 * attendance reconciliation (`replaceFileLinks`) needs the unscoped form.
 */
export async function fileLinkOwnership(engine: BrainEngine, sourceId: string, scope?: { slugs: readonly string[] }) {
  const pages = scope
    ? await engine.executeRaw<LinkPageMetadata>(`SELECT slug, source_id, type, title, frontmatter->'aliases' AS aliases, knowledge_revision
        FROM pages WHERE deleted_at IS NULL AND source_id = $1 AND slug = ANY($2::text[]) ORDER BY slug`, [sourceId, [...scope.slugs]])
    : await loadLinkPageMetadata(engine);
  const metadata = new Map(pages.map(row => [`${row.source_id}\0${row.slug}`, row]));
  const resolver = makeIndexedLinkResolver(pages, sourceId);
  const { allSlugs, slugToSources } = indexLinkSources(pages);
  const policy = await loadLinkSourcePolicy(engine, sourceId);
  const resolve = (slug: string, candidate: LinkCandidate) => resolveCandidateSources(candidate, slug, sourceId,
    allSlugs, slugToSources, policy.allowCrossSource, policy);
  const origins = await engine.executeRaw<{ slug: string }>(`SELECT DISTINCT o.slug FROM links l
    JOIN pages o ON o.id=l.origin_page_id WHERE o.source_id=$1 AND l.link_source IN ('markdown','wikilink-resolved','frontmatter')
      AND l.link_type='attended' AND l.to_page_id=l.origin_page_id${scope ? ' AND o.slug = ANY($2::text[])' : ''}`,
    scope ? [sourceId, [...scope.slugs]] : [sourceId]);
  return { metadata, resolver, resolve, origins: new Set(origins.map(row => row.slug)) };
}

export function indexLinkSources(pages: ReadonlyArray<{ slug: string; source_id: string }>) {
  const allSlugs = new Set<string>();
  const slugToSources = new Map<string, string[]>();
  for (const page of pages) {
    allSlugs.add(page.slug);
    const sources = slugToSources.get(page.slug) ?? [];
    sources.push(page.source_id);
    slugToSources.set(page.slug, sources);
  }
  return { allSlugs, slugToSources };
}

export interface LinkSourcePolicy { allowCrossSource: boolean; crossSource: boolean; defaultSourceId: string }

export async function loadLinkSourcePolicy(engine: BrainEngine, sourceId: string): Promise<LinkSourcePolicy> {
  const [source, crossSource, defaultSourceId] = await Promise.all([
    fetchSource(engine, sourceId), isCrossSourceLinksEnabled(engine), resolveLinkFallbackDefault(engine),
  ]);
  return { allowCrossSource: source !== null && !source.archived && isSourceFederated(source.config), crossSource, defaultSourceId };
}

export type LinkCandidateAdmission = { ok: true; row: LinkBatchInput } | Exclude<CandidateSourceResolution, { ok: true }>;

/**
 * #4680: the one source-policy resolver for one origin page, shared by put_page's link
 * preparation and the source reconciliation sweep (`extract links --source db` applies the
 * same `resolveCandidateSources` contract inline). `resolve` decides both endpoints'
 * sources under the policy; `admit` turns an approved candidate into its link row and
 * requires only that an outbound edge originates in the origin's own source (publication
 * enforces that too): a target the policy resolved into another source is admitted, a
 * candidate the policy refused is a counted drop. Canonical attendance is the one edge
 * whose far endpoint, the person, may live in another source.
 */
export function makeLinkSourceResolver(pageSlug: string, pageSourceId: string,
  endpoints: { allSlugs: Set<string>; slugToSources: Map<string, string[]> }, policy: LinkSourcePolicy) {
  const resolve = (candidate: LinkCandidate): CandidateSourceResolution => resolveCandidateSources(candidate, pageSlug, pageSourceId,
    endpoints.allSlugs, endpoints.slugToSources, policy.allowCrossSource, policy);
  return {
    resolve,
    crossSourceAllowed: policy.allowCrossSource || policy.crossSource,
    admit(candidate: LinkCandidate): LinkCandidateAdmission {
      const resolved = resolve(candidate);
      if (!resolved.ok) return resolved;
      if (!candidate.canonicalAttendance && resolved.fromSourceId !== pageSourceId) return { ok: false, reason: 'cross_source' };
      return { ok: true, row: resolvedLinkCandidate(candidate, pageSlug, pageSourceId, resolved) };
    },
  };
}

/**
 * Endpoint fences for one origin's link batch: every endpoint at the revision
 * the resolver census captured. The source-qualified `origin`, when given, is
 * left out (#6474): its body and type came from the snapshot whose revision
 * `expectedRevision` already fences at publication, so fencing it again at
 * the older census revision would defer a page rewritten between the census
 * and its own read even though the run extracted its live body. A page with
 * the same slug in another source is still fenced. This is the one place the
 * rule lives; the managed preparation and both DB walks call it.
 */
export function capturedLinkEndpoints(links: LinkBatchInput[], metadata: ReadonlyMap<string, Pick<LinkPageMetadata, 'slug' | 'source_id' | 'knowledge_revision'>>,
  origin?: { slug: string; sourceId: string }) {
  const keys = new Set(links.flatMap(link => [
    `${link.from_source_id ?? 'default'}\0${link.from_slug}`, `${link.to_source_id ?? 'default'}\0${link.to_slug}`,
  ]));
  if (origin) keys.delete(`${origin.sourceId}\0${origin.slug}`);
  return [...keys].map(key => {
    const endpoint = metadata.get(key);
    if (!endpoint) throw new Error('A derived link endpoint was not captured during type resolution');
    return { slug: endpoint.slug, sourceId: endpoint.source_id, revision: endpoint.knowledge_revision };
  });
}

/**
 * v0.42.7 (#1696): pure cross-source resolution for one extracted link
 * candidate. Validates both endpoints exist (else the batch JOIN drops the row),
 * then picks from_source_id / to_source_id: prefer the origin page's source,
 * fall back to 'default', else skip (never push a wrong-source edge). Shared
 * by extractLinksFromDB and extractStaleFromDB so the F10 multi-source
 * resolution and the source-isolation policy can't drift.
 *
 * v0.46.28.0 (#2589): the failure case now carries a `reason` instead of a
 * bare `null`. A target that resolves via `global_basename` to a page that
 * exists ONLY in a source other than the origin's or 'default' was
 * indistinguishable from a genuinely-missing target — both silently dropped
 * the candidate and both counted (misleadingly) as "target page doesn't
 * exist". This stays default-deny by design: cross-source edges remain
 * unwritten (source isolation — see CLAUDE.md), but callers can now
 * attribute the drop correctly instead of reporting a wrong reason.
 *
 * #3478: only a federated origin source may fall back to 'default'; when
 * `allowCrossSource` is false both endpoints must live in the page's own
 * source, else skip with reason 'cross_source' (never push a wrong-source
 * edge).
 *
 * #3908: `opts.crossSource` (the `link_resolution.cross_source` config flag)
 * is the explicit operator opt-in — it supersedes the ambient federation
 * gate, which only exists to stop DEFAULT-ON silent cross-source regrowth.
 * With it on, a cross-source-only candidate resolves with the
 * lexicographically smallest matching source (deterministic, so repeated
 * extracts and both engines converge on the same edge under the
 * (source_id, slug) composite key) instead of dropping with 'cross_source'.
 *
 * #4611: the fallback lane compares against `opts.defaultSourceId` (the
 * configured `sources.default`, resolved once per run by the callers via
 * resolveLinkFallbackDefault) instead of the LITERAL string 'default'.
 * Renaming the brain's default source no longer silently kills the
 * cross-source fallback. Omitted → 'default' (back-compat).
 */
export type CandidateSourceResolution =
  | { ok: true; fromSlug: string; fromSourceId: string; toSourceId: string }
  | { ok: false; reason: 'missing_target' | 'missing_from' | 'cross_source' };

/**
 * #4611: resolve the source id the cross-source link fallback compares
 * against. Reads the operator-configured `sources.default` (same key the
 * write-routing ladder in source-resolver.ts tier 5 reads), silently
 * falling back to the seeded literal 'default' on unset/invalid/config
 * errors — extraction must never fail on a bad config row.
 */
export async function resolveLinkFallbackDefault(
  engine: Pick<BrainEngine, 'getConfig'>,
): Promise<string> {
  try {
    const v = await engine.getConfig('sources.default');
    if (v && isValidSourceId(v)) return v;
  } catch {
    // Best-effort read; the seeded literal below is the safe terminal.
  }
  return 'default';
}

export function resolveCandidateSources(
  c: LinkCandidate,
  pageSlug: string,
  pageSourceId: string,
  allSlugs: Set<string>,
  slugToSources: Map<string, string[]>,
  allowCrossSource: boolean,
  opts: { crossSource?: boolean; defaultSourceId?: string } = {},
): CandidateSourceResolution {
  const fromSlug = c.fromSlug ?? pageSlug;
  if (!allSlugs.has(c.targetSlug)) return { ok: false, reason: 'missing_target' };
  if (!allSlugs.has(fromSlug)) return { ok: false, reason: 'missing_from' };
  const fromSources = slugToSources.get(fromSlug) ?? [];
  const targetSources = slugToSources.get(c.targetSlug) ?? [];
  if (c.targetSourceId) {
    if (!targetSources.includes(c.targetSourceId)) return { ok: false, reason: 'missing_target' };
    if (!fromSources.includes(pageSourceId)) return { ok: false, reason: 'missing_from' };
    if (c.targetSourceId !== pageSourceId && !allowCrossSource && !opts.crossSource) return { ok: false, reason: 'cross_source' };
    return { ok: true, fromSlug, fromSourceId: pageSourceId, toSourceId: c.targetSourceId };
  }
  if (!allowCrossSource && !opts.crossSource) {
    if (!fromSources.includes(pageSourceId) || !targetSources.includes(pageSourceId)) {
      // #3478 isolation × #2589 counting: both endpoints exist but not in
      // the origin's own source — a counted cross-source drop, never an edge.
      return { ok: false, reason: 'cross_source' };
    }
    return { ok: true, fromSlug, fromSourceId: pageSourceId, toSourceId: pageSourceId };
  }
  // #4611: follow the CONFIGURED default source, not the literal 'default'.
  const defaultSourceId = opts.defaultSourceId ?? 'default';
  const fromSourceId = fromSources.includes(pageSourceId) ? pageSourceId
    : (fromSources.includes(defaultSourceId) ? defaultSourceId : fromSources[0]);
  let toSourceId: string;
  if (targetSources.includes(fromSourceId)) {
    toSourceId = fromSourceId;
  } else if (targetSources.includes(defaultSourceId)) {
    toSourceId = defaultSourceId;
  } else if (targetSources.length > 0) {
    // #2589: the target exists ONLY in other sources. Historically this was
    // a silent null (indistinguishable from a missing endpoint — multi-source
    // graphs went sparse with dead_links stuck at 0). Behind the opt-in
    // `link_resolution.cross_source` flag the edge is allowed with a
    // DETERMINISTIC pick (lexicographically smallest source, so repeated
    // extracts and both engines converge on the same edge under the
    // (source_id, slug) composite key); off, callers get the distinguishable
    // 'cross_source' reason to COUNT the drop instead of burying it.
    if (!opts.crossSource) return { ok: false, reason: 'cross_source' };
    // Allocation-free deterministic min (bulk loops call this per candidate;
    // in the motivating federated topology most candidates hit this branch).
    let min = targetSources[0];
    for (const s of targetSources) if (s < min) min = s;
    toSourceId = min;
  } else {
    return { ok: false, reason: 'cross_source' };
  }
  return { ok: true, fromSlug, fromSourceId, toSourceId };
}

/**
 * (slug, source_id) refs for EXACTLY the given slugs, chunked IN-list —
 * the bounded replacement for listAllPageRefs in the sweep's pass 2. Same
 * visibility as listAllPageRefs (deleted_at IS NULL).
 */
export async function lookupRefsForSlugs(
  engine: BrainEngine,
  slugs: string[],
): Promise<{ allSlugs: Set<string>; slugToSources: Map<string, string[]>;
  metadata: Array<{ slug: string; source_id: string; type: string; knowledge_revision: string }> }> {
  const allSlugs = new Set<string>();
  const slugToSources = new Map<string, string[]>();
  const metadata: Array<{ slug: string; source_id: string; type: string; knowledge_revision: string }> = [];
  const CHUNK = 200;
  for (let i = 0; i < slugs.length; i += CHUNK) {
    const chunk = slugs.slice(i, i + CHUNK);
    const placeholders = chunk.map((_, j) => `$${j + 1}`).join(', ');
    const rows = await engine.executeRaw<{ slug: string; source_id: string; type: string; knowledge_revision: string }>(
      `SELECT slug, source_id, type, knowledge_revision FROM pages
        WHERE deleted_at IS NULL AND slug IN (${placeholders})`,
      chunk,
    );
    metadata.push(...rows);
    for (const ref of rows) {
      allSlugs.add(ref.slug);
      const list = slugToSources.get(ref.slug) ?? [];
      list.push(ref.source_id);
      slugToSources.set(ref.slug, list);
    }
  }
  return { allSlugs, slugToSources, metadata };
}