Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.1 KB
Raw Blame
/**
 * #4224 — cross-source entity identity (federation v1).
 *
 * THE IDENTITY KEY IS (source_id, slug). A brain can mount many sources and
 * the same real-world entity routinely exists as a different page in each
 * (`wiki:people/alice` vs `team-brain:people/alice-chen`). Nothing else in
 * gbrain asserts "these pages are the same entity" — slugs are only unique
 * per source, and titles collide freely. The `entity_identities` table
 * (migration v137) records that assertion explicitly: member pages grouped
 * under an opaque `entity_id` handle, one optional canonical member per
 * group.
 *
 * v1 posture (deliberate):
 *   - MANUAL-ONLY. Rows come exclusively from the entity_identity_link op.
 *     No auto-matching, no name-similarity heuristics — a wrong identity
 *     merge silently corrupts retrieval, so v1 keeps a human in the loop.
 *     Near names are only flagged for review (entities/near-name.ts,
 *     extraction_pending possible_duplicates); nothing here is written from them.
 *   - A page belongs to at most one identity (UNIQUE (source_id, page_id));
 *     re-linking MOVES it.
 *   - Retrieval union is OFF by default, gated by the
 *     `entity_identity.union` config key. When on, the get_links /
 *     get_backlinks ops merge edges from a page's identity co-members
 *     (dedup'd), never widening past the caller's source grant.
 *
 * Everything here goes through `engine.executeRaw` with the SAME SQL text on
 * both engines — parity by construction (pinned in the entity-identity test
 * + the DATABASE_URL-gated engine-parity suite).
 */

import type { BrainEngine } from './engine.ts';
import type { Link } from './types.ts';
import { isMissingSchemaObject } from './utils.ts';
import { privatePagesFilterForCaller } from './search/private-visibility.ts';

/** Config key for the flag-gated retrieval union. Default OFF. */
export const ENTITY_IDENTITY_UNION_CONFIG_KEY = 'entity_identity.union';

/**
 * Opaque identity handle grammar. Slug-flavored so handles read well in CLI
 * output and can't smuggle whitespace/quotes into messages.
 */
const ENTITY_ID_RE = /^[a-z0-9][a-z0-9._/-]{0,127}$/;

export function validateEntityId(entityId: string): string {
  const trimmed = entityId.trim();
  if (!ENTITY_ID_RE.test(trimmed)) {
    throw new Error(
      `invalid entity_id '${entityId}': expected lowercase [a-z0-9._/-], starting alphanumeric, max 128 chars`,
    );
  }
  return trimmed;
}

export interface EntityIdentityMember {
  entity_id: string;
  source_id: string;
  slug: string;
  title: string | null;
  confidence: number;
  established_by: string;
  established_at: Date | string;
  canonical: boolean;
}

async function resolvePageId(
  engine: BrainEngine,
  slug: string,
  sourceId: string,
): Promise<number> {
  const rows = await engine.executeRaw<{ id: number }>(
    `SELECT id FROM pages WHERE slug = $1 AND source_id = $2 AND deleted_at IS NULL`,
    [slug, sourceId],
  );
  if (rows.length === 0) {
    throw new Error(`page not found: ${slug} (source ${sourceId}) — the identity key is (source_id, slug)`);
  }
  return Number(rows[0]!.id);
}

/**
 * Link a page into an identity group (upsert on the page: re-linking MOVES
 * the page to the new identity — explicit manual intent). `canonical: true`
 * demotes the group's previous canonical member first, in the same
 * transaction as the link, so a failed link never leaves the group without
 * its canonical. Omitting `canonical` keeps a re-linked member's canonical
 * flag within the same group (a confidence update must not demote it);
 * `canonical: false` demotes explicitly. The partial unique index is the
 * backstop against a race leaving two canonicals.
 */
export async function linkEntityIdentity(
  engine: BrainEngine,
  opts: {
    entityId: string;
    slug: string;
    sourceId: string;
    confidence?: number;
    establishedBy?: string;
    canonical?: boolean;
  },
): Promise<EntityIdentityMember> {
  const entityId = validateEntityId(opts.entityId);
  const confidence = opts.confidence ?? 1.0;
  if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
    throw new Error(`confidence must be in [0,1], got ${opts.confidence}`);
  }
  const establishedBy = (opts.establishedBy ?? 'manual').trim() || 'manual';
  const canonical = typeof opts.canonical === 'boolean' ? opts.canonical : null;
  const pageId = await resolvePageId(engine, opts.slug, opts.sourceId);

  await engine.transaction(async (tx) => {
    if (canonical === true) {
      await tx.executeRaw(
        `UPDATE entity_identities SET canonical = false WHERE entity_id = $1 AND canonical`,
        [entityId],
      );
    }
    await tx.executeRaw(
      `INSERT INTO entity_identities (entity_id, source_id, page_id, confidence, established_by, canonical)
       VALUES ($1, $2, $3, $4, $5, COALESCE($6::boolean, false))
       ON CONFLICT (source_id, page_id) DO UPDATE SET
         entity_id = EXCLUDED.entity_id,
         confidence = EXCLUDED.confidence,
         established_by = EXCLUDED.established_by,
         canonical = CASE WHEN $6::boolean IS NULL
           THEN entity_identities.canonical AND entity_identities.entity_id = EXCLUDED.entity_id
           ELSE EXCLUDED.canonical END,
         established_at = now()`,
      [entityId, opts.sourceId, pageId, confidence, establishedBy, canonical],
    );
  });

  const members = await listEntityIdentities(engine, { entityId });
  const me = members.find(m => m.slug === opts.slug && m.source_id === opts.sourceId);
  // The row was just written; me is only absent if a concurrent delete raced us.
  if (!me) throw new Error(`identity link for ${opts.slug} did not persist`);
  return me;
}

/** Remove a page from an identity group. Returns true when a row was removed. */
export async function unlinkEntityIdentity(
  engine: BrainEngine,
  opts: { entityId: string; slug: string; sourceId: string },
): Promise<boolean> {
  const entityId = validateEntityId(opts.entityId);
  const rows = await engine.executeRaw<{ id: number }>(
    `DELETE FROM entity_identities ei
     USING pages p
     WHERE ei.page_id = p.id
       AND ei.entity_id = $1
       AND p.slug = $2
       AND ei.source_id = $3
     RETURNING ei.id`,
    [entityId, opts.slug, opts.sourceId],
  );
  return rows.length > 0;
}

/**
 * List identity members. Filters compose (AND). Non-empty `allowedSources` restricts
 * MEMBER VISIBILITY (federated read grant) — a caller who can't read source
 * X never learns X's member pages, even when another member matched. Without
 * that grant, `sourceId` is the scalar floor; private seeds and members are
 * excluded together when `excludePrivate` is set.
 *
 * The identity key is (source_id, slug), so the `slug` filter alone is
 * ambiguous: pass `slugSourceId` to seed group resolution from exactly the
 * (slug, source) page the caller is reading. Without it, the seed sub-select
 * is scoped to `allowedSources` when provided (a caller must not discover a
 * group through a seed page outside their grant), and only a trusted
 * unscoped caller (neither given) seeds from any source.
 */
export async function listEntityIdentities(
  engine: BrainEngine,
  opts: { entityId?: string; slug?: string; slugSourceId?: string; sourceId?: string; allowedSources?: string[]; excludePrivate?: boolean } = {},
): Promise<EntityIdentityMember[]> {
  const where: string[] = [];
  const params: unknown[] = [];
  const allowedSources = opts.allowedSources?.length ? opts.allowedSources
    : opts.sourceId ? [opts.sourceId] : undefined;
  if (opts.excludePrivate) where.push(privatePagesFilterForCaller('p'));
  if (opts.entityId) {
    params.push(validateEntityId(opts.entityId));
    where.push(`ei.entity_id = $${params.length}`);
  }
  if (opts.slug) {
    params.push(opts.slug);
    const slugParam = params.length;
    // #4224 review fix: the seed sub-select must be source-scoped — an
    // unscoped `p2.slug = $` matches ANY source's page with that slug, so a
    // read against a NON-member page unioned a foreign same-slug member's
    // group.
    let seedScope = '';
    if (opts.slugSourceId) {
      params.push(opts.slugSourceId);
      seedScope = ` AND ei2.source_id = $${params.length}`;
    }
    if (allowedSources) {
      const ph = allowedSources.map((s) => {
        params.push(s);
        return `$${params.length}`;
      });
      seedScope += ` AND ei2.source_id IN (${ph.join(', ')})`;
    }
    if (opts.excludePrivate) seedScope += ` AND ${privatePagesFilterForCaller('p2')}`;
    where.push(`ei.entity_id IN (
      SELECT ei2.entity_id FROM entity_identities ei2
      JOIN pages p2 ON p2.id = ei2.page_id AND p2.source_id = ei2.source_id
      WHERE p2.slug = $${slugParam} AND p2.deleted_at IS NULL${seedScope}
    )`);
  }
  if (allowedSources) {
    const placeholders = allowedSources.map((s) => {
      params.push(s);
      return `$${params.length}`;
    });
    where.push(`ei.source_id IN (${placeholders.join(', ')})`);
  }
  try {
    const rows = await engine.executeRaw<{
      entity_id: string;
      source_id: string;
      slug: string;
      title: string | null;
      confidence: number;
      established_by: string;
      established_at: Date | string;
      canonical: boolean;
    }>(
      `SELECT ei.entity_id, ei.source_id, p.slug, p.title, ei.confidence,
              ei.established_by, ei.established_at, ei.canonical
       FROM entity_identities ei
       JOIN pages p ON p.id = ei.page_id AND p.source_id = ei.source_id AND p.deleted_at IS NULL
       ${where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''}
       ORDER BY ei.entity_id, ei.canonical DESC, ei.source_id, p.slug`,
      params,
    );
    return rows.map(r => ({
      entity_id: r.entity_id,
      source_id: r.source_id,
      slug: r.slug,
      title: r.title,
      confidence: Number(r.confidence),
      established_by: r.established_by,
      established_at: r.established_at,
      canonical: r.canonical === true,
    }));
  } catch (e) {
    // Pre-v137 brain (table missing): identity is simply "not set up yet".
    if (isMissingSchemaObject(e, 'table', 'entity_identities')) return [];
    throw e;
  }
}

/**
 * The identity group of each given (source_id, slug) page that belongs to one,
 * keyed `${source_id}:${slug}`. Pages outside any group are absent.
 */
export async function identityIdsForPages(
  engine: BrainEngine,
  pages: Array<{ sourceId: string; slug: string }>,
): Promise<Map<string, string>> {
  if (pages.length === 0) return new Map();
  const wanted = new Set(pages.map(p => `${p.sourceId}:${p.slug}`));
  try {
    const rows = await engine.executeRaw<{ entity_id: string; source_id: string; slug: string }>(
      `SELECT ei.entity_id, ei.source_id, p.slug
         FROM entity_identities ei
         JOIN pages p ON p.id = ei.page_id AND p.source_id = ei.source_id AND p.deleted_at IS NULL
        WHERE ei.source_id = ANY($1::text[]) AND p.slug = ANY($2::text[])`,
      [[...new Set(pages.map(p => p.sourceId))], [...new Set(pages.map(p => p.slug))]],
    );
    return new Map(rows.map((r): [string, string] => [`${r.source_id}:${r.slug}`, r.entity_id]).filter(([k]) => wanted.has(k)));
  } catch (e) {
    if (isMissingSchemaObject(e, 'table', 'entity_identities')) return new Map();
    throw e;
  }
}

/** Is the flag-gated retrieval union on? Fail-closed on any read error. */
export async function isIdentityUnionEnabled(engine: BrainEngine): Promise<boolean> {
  try {
    const v = await engine.getConfig(ENTITY_IDENTITY_UNION_CONFIG_KEY);
    if (v === null || v === undefined) return false;
    const s = String(v).trim().toLowerCase();
    return s === 'true' || s === '1';
  } catch {
    return false;
  }
}

/**
 * Flag-gated retrieval union for the link read ops (#4224 v1 union surface;
 * chunk-side union is a filed follow-up — member pages' chunks already
 * surface independently in search, so edges are where identity is invisible
 * today).
 *
 * Given a page slug and the links already fetched for it, fetch the same
 * direction's links for every identity CO-MEMBER the caller may see and
 * merge (dedup'd on the full edge key). Sources outside `allowedSources`
 * (when provided) are never consulted — the union must not widen a
 * federated caller's grant.
 *
 * Pure no-op when the flag is off, the table is missing, or the page is not
 * an identity member.
 */
export async function unionLinksAcrossIdentity(
  engine: BrainEngine,
  slug: string,
  links: Link[],
  direction: 'out' | 'in',
  opts: {
    sourceId?: string; allowedSources?: string[]; excludePrivate?: boolean; temporal?: import('./link-validity.ts').EdgeTemporalOpts;
    /** #6876: the caller's keyset page, applied to each member read; `onMemberRead` sees each member's rows before the merge. */
    page?: import('./link-validity.ts').LinkReadPage; onMemberRead?: (rows: Link[]) => void;
  } = {},
): Promise<Link[]> {
  if (!(await isIdentityUnionEnabled(engine))) return links;
  let members: EntityIdentityMember[];
  try {
    members = await listEntityIdentities(engine, {
      slug,
      // #4224 review fix: seed group resolution from the BASE page's
      // (slug, source) — never from a foreign same-slug page.
      slugSourceId: opts.sourceId,
      sourceId: opts.sourceId,
      allowedSources: opts.allowedSources,
      excludePrivate: opts.excludePrivate,
    });
  } catch {
    return links; // never let the union break the base read
  }
  // #4224 review fix: the identity key is (source_id, slug) — exclude only
  // the BASE pair. A same-slug co-member in ANOTHER source is a real
  // co-member whose edges the scalar-scoped base read did not fetch. When
  // the base source is unknown (trusted unscoped / federated multi-grant),
  // the base read already spanned every visible same-slug page, so
  // slug-level exclusion is exact there.
  const coMembers = opts.sourceId
    ? members.filter(m => !(m.slug === slug && m.source_id === opts.sourceId))
    : members.filter(m => m.slug !== slug);
  if (coMembers.length === 0) return links;

  const merged: Link[] = [...links];
  const keyOf = (l: Link) =>
    `${l.from_source_id}:${l.from_slug}|${l.to_source_id}:${l.to_slug}|${l.link_type}|${l.link_source ?? ''}`;
  const seen = new Set(merged.map(keyOf));
  for (const m of coMembers) {
    try {
      const memberScope = {
        sourceId: m.source_id,
        ...(opts.allowedSources?.length ? { sourceIds: opts.allowedSources } : {}),
        excludePrivate: opts.excludePrivate,
        ...(opts.temporal ? { temporal: opts.temporal } : {}),
        ...(opts.page ? { page: opts.page } : {}),
      };
      const memberLinks = direction === 'out'
        ? await engine.getLinks(m.slug, memberScope)
        : await engine.getBacklinks(m.slug, memberScope);
      opts.onMemberRead?.(memberLinks);
      for (const l of memberLinks) {
        // Federated engine reads span same-slug pages across the grant. Keep
        // this identity member's concrete pair, not a same-slug non-member.
        const memberSource = direction === 'out' ? l.from_source_id : l.to_source_id;
        if (memberSource !== m.source_id) continue;
        const k = keyOf(l);
        if (seen.has(k)) continue;
        seen.add(k);
        merged.push(l);
      }
    } catch {
      // A single member's failure never breaks the base read.
    }
  }
  return merged;
}