Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(links): count hidden former relationships in SQL instead of fetching them (#7075)
garrytan32257eb5362d
diff --git a/src/core/engine-sql/links.ts b/src/core/engine-sql/links.tsindex b17b2d165f..843aa9b270 100644--- a/src/core/engine-sql/links.ts+++ b/src/core/engine-sql/links.ts@@ -28,10 +28,16 @@ import type { LinkReadScope } from '../link-validity.ts';/** Bounded link reads annotate rows with relationship status (never filter) when a temporal read is requested. */function finishTemporal(rows: unknown[], opts?: LinkReadScope): Link[] {- const t = opts?.temporal;+ const t = opts?.page?.count ? undefined : opts?.temporal;return (t ? rows.map(r => annotateTemporalRow(r as Link, t.asOf)) : rows) as Link[];}+/** Run one link read; a counting page (#7075) runs it as count(*) and returns one row carrying `link_total`. */+async function runLinkRead(exec: ScopedRead, opts: LinkReadScope | undefined, statement: SqlFragment): Promise<unknown[]> {+ if (!opts?.page?.count) return (await exec.run(statement)).rows;+ return (await exec.run(sqlFragment`SELECT count(*)::int AS link_total FROM (${statement}) counted`)).rows;+}+/** WHERE fragment that keeps only edges matching the caller's temporal policy (walks and fanout). */function temporalWalkFilter(alias: string, opts?: LinkReadScope) {return opts?.temporal ? trustedSql(`AND ${relationshipFilterSql(alias, { ...opts.temporal, excludePrivate: opts.excludePrivate })}`) : sqlFragment``;@@ -355,7 +361,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc// out of grant. Remote MCP clients always land here.if (opts?.sourceIds && opts.sourceIds.length > 0) {const ids = opts.sourceIds;- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -368,7 +374,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadScWHERE f.slug = ${slug} AND f.source_id = ANY(${ids}::text[]) AND t.source_id = ANY(${ids}::text[])AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}// v0.31.8 (D16) + #2200: the federated arm above is the first branch; the@@ -379,7 +385,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc// NULL) so links to/from soft-deleted pages stop voting in the graph,// matching orphans/get/list/search visibility.if (opts?.sourceId) {- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -392,10 +398,10 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadScWHERE f.slug = ${slug} AND f.source_id = ${opts.sourceId}AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -408,7 +414,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadScWHERE f.slug = ${slug}AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}@@ -424,7 +430,7 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkRe// a foreign referrer nor a foreign origin slug is disclosed to the caller.if (opts?.sourceIds && opts.sourceIds.length > 0) {const ids = opts.sourceIds;- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -437,13 +443,13 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkReWHERE t.slug = ${slug} AND t.source_id = ANY(${ids}::text[]) AND f.source_id = ANY(${ids}::text[])AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}// v0.31.8 (D16) + #2200: federated arm above is first; two below mirror getLinks// (incl. the #3754 soft-delete endpoint filter on all three arms).if (opts?.sourceId) {- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -456,10 +462,10 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkReWHERE t.slug = ${slug} AND t.source_id = ${opts.sourceId}AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}- const rows = (await exec.run(sqlFragment`+ const rows = await runLinkRead(exec, opts, sqlFragment`SELECT f.slug as from_slug, f.source_id as from_source_id,t.slug as to_slug, t.source_id as to_source_id,l.link_type, ${page.context}, l.link_source,@@ -472,7 +478,7 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkReWHERE t.slug = ${slug}AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}ORDER BY l.id${page.limit}- `)).rows;+ `);return finish(rows);}diff --git a/src/core/link-validity.ts b/src/core/link-validity.tsindex 218c413c3e..20f047e5a6 100644--- a/src/core/link-validity.ts+++ b/src/core/link-validity.ts@@ -398,6 +398,8 @@ export interface LinkReadPage {relationship?: { policy: EdgeTemporalOpts; keep: 'match' | 'hidden' };/** Context longer than this is cut, ending in an ellipsis. */contextChars?: number;+ /** Count the rows in SQL instead of returning them: the read returns one row whose `link_total` is the count. */+ count?: boolean;}export interface EdgeTemporalOpts {diff --git a/src/core/ops/links.ts b/src/core/ops/links.tsindex ce5a92e7a2..50c97c1e5b 100644--- a/src/core/ops/links.ts+++ b/src/core/ops/links.ts@@ -351,7 +351,10 @@ async function readLinkEdges(do {const { streams, rows } = await read({ ...base, afterId, limit: listing.limit + 1, ...(filtering ? { relationship: { policy: temporal, keep: 'match' as const } } : {}) });const page = cutLinkPage(streams, rows, listing.limit);- if (filtering) hidden += (await read({ ...base, afterId, throughId: page.nextId, relationship: { policy: temporal, keep: 'hidden' } })).rows.length;+ if (filtering) {+ const counted = await read({ ...(linkType ? { linkType } : {}), ...(fromType ? { fromType } : {}), afterId, throughId: page.nextId, relationship: { policy: temporal, keep: 'hidden' }, count: true });+ for (const stream of counted.streams) hidden += Number((stream[0] as Link & { link_total?: number } | undefined)?.link_total ?? 0);+ }const shaped = filterTemporalLinks(page.rows, temporal);kept.push(...shaped.kept);hidden += shaped.hidden;diff --git a/test/link-hidden-count-7075.test.ts b/test/link-hidden-count-7075.test.tsnew file mode 100644index 0000000000..9a0cbb60ea--- /dev/null+++ b/test/link-hidden-count-7075.test.ts@@ -0,0 +1,140 @@+/**+ * #7075 — get_links / get_backlinks count the former relationships a page's+ * temporal policy hides with a count(*) in SQL. Regression: the last page of+ * a listing fetched every hidden edge (context included) only to count them.+ * The count reported in `gbrain.temporal.hidden` stays the same; no hidden+ * edge row leaves the database.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { dispatchToolCall } from '../src/mcp/dispatch.ts';+import { ENTITY_IDENTITY_UNION_CONFIG_KEY, linkEntityIdentity } from '../src/core/entity-identity.ts';+import type { LinkReadScope } from '../src/core/link-validity.ts';+import type { Link } from '../src/core/types.ts';++const config = { engine: 'pglite' } as never;+const LOCAL = { remote: false, sourceId: 'default', config };+const REMOTE = { remote: true, transport: 'stdio' as const, sourceId: 'default', takesHoldersAllowList: ['world'], config };+const COMPANIES = ['companies/c0', 'companies/c1', 'companies/c2', 'companies/c3'];++let engine: PGLiteEngine;+const reads: Array<{ opts?: LinkReadScope; rows: number }> = [];++async function put(slug: string, type = 'note') {+ await engine.putPage(slug, { type: type as never, title: slug, compiled_truth: `${slug} body`, frontmatter: {} }, { sourceId: 'default' });+}++async function ended(from: string, to: string) {+ const r = await dispatchToolCall(engine, 'add_link', { from, to, link_type: 'works_at', valid_from: '2019-01-01', valid_until: '2020-01-01' }, LOCAL as never);+ if (r.isError) throw new Error(`seed add_link failed: ${r.content[0].text}`);+}++async function call(op: string, args: Record<string, unknown>, opts: Record<string, unknown>) {+ reads.length = 0;+ const r = await dispatchToolCall(engine, op, args, opts as never);+ if (r.isError) throw new Error(r.content[0].text);+ const meta = (r as { _meta?: Record<string, unknown> })._meta ?? {};+ return { rows: JSON.parse(r.content[0].text) as Array<Record<string, unknown>>, hidden: (meta['gbrain.temporal'] as { hidden?: number } | undefined)?.hidden };+}++const hiddenReads = () => reads.filter(r => r.opts?.page?.relationship?.keep === 'hidden');++beforeAll(async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+ for (const slug of ['hubs/alpha', 'hubs/beta', 'idn/a', 'idn/b', 'targets/t0', 'targets/t1', 'targets/t2']) await put(slug);+ for (const slug of COMPANIES) await put(slug, 'company');+ // hubs/alpha: two live edges, then three ended works_at (all on the last page).+ await engine.addLink('hubs/alpha', 'targets/t0', 'one', 'k000', 'manual');+ await engine.addLink('hubs/alpha', 'targets/t1', 'two', 'k000', 'manual');+ for (const c of COMPANIES.slice(0, 3)) await ended('hubs/alpha', c);+ // hubs/beta: hidden edges between and after live edges, for a paged read.+ await engine.addLink('hubs/beta', 'targets/t0', 'one', 'k000', 'manual');+ await ended('hubs/beta', COMPANIES[0]);+ await engine.addLink('hubs/beta', 'targets/t1', 'two', 'k000', 'manual');+ await ended('hubs/beta', COMPANIES[1]);+ await engine.addLink('hubs/beta', 'targets/t2', 'three', 'k000', 'manual');+ await ended('hubs/beta', COMPANIES[2]);+ await ended('hubs/beta', COMPANIES[3]);+ // Identity union: idn/a and idn/b are one entity; each carries hidden edges+ // (two of them into companies/c0, which the backlinks case also counts).+ await ended('idn/a', COMPANIES[0]);+ await ended('idn/b', COMPANIES[0]);+ await ended('idn/a', COMPANIES[1]);+ await ended('idn/b', COMPANIES[2]);+ await engine.addLink('idn/a', 'targets/t0', 'a live', 'k000', 'manual');+ await engine.setConfig(ENTITY_IDENTITY_UNION_CONFIG_KEY, 'true');+ await linkEntityIdentity(engine, { entityId: 'thing/idn', slug: 'idn/a', sourceId: 'default' });+ await linkEntityIdentity(engine, { entityId: 'thing/idn', slug: 'idn/b', sourceId: 'default' });++ const getLinks = engine.getLinks.bind(engine);+ const getBacklinks = engine.getBacklinks.bind(engine);+ engine.getLinks = async (slug: string, opts?: LinkReadScope): Promise<Link[]> => {+ const rows = await getLinks(slug, opts);+ reads.push({ opts, rows: rows.length });+ return rows;+ };+ engine.getBacklinks = async (slug: string, opts?: LinkReadScope): Promise<Link[]> => {+ const rows = await getBacklinks(slug, opts);+ reads.push({ opts, rows: rows.length });+ return rows;+ };+}, 120_000);++afterAll(async () => { if (engine) await engine.disconnect(); }, 60_000);++describe('#7075 hidden-relationship count', () => {+ test('the last page counts its hidden edges in SQL and reports the same count', async () => {+ for (const opts of [LOCAL, REMOTE]) {+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha' }, opts);+ expect(rows.map(r => r.context)).toEqual(['one', 'two']);+ expect(hidden).toBe(3);+ expect(hiddenReads().length).toBeGreaterThan(0);+ for (const r of hiddenReads()) {+ expect(r.opts?.page?.count).toBe(true);+ expect(r.rows).toBe(1);+ }+ }+ });++ test('a paged listing counts each page\'s hidden edges once, in SQL', async () => {+ const first = await call('get_links', { slug: 'hubs/beta', limit: 2 }, REMOTE);+ expect(first.rows.map(r => r.context)).toEqual(['one', 'two']);+ expect(first.hidden).toBe(1);+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);+ const second = await call('get_links', { slug: 'hubs/beta', limit: 2, cursor: first.rows[1].next_cursor }, REMOTE);+ expect(second.rows.map(r => r.context)).toEqual(['three']);+ expect(second.hidden).toBe(3);+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);+ });++ test('get_backlinks counts the same way', async () => {+ const { rows, hidden } = await call('get_backlinks', { slug: COMPANIES[0] }, REMOTE);+ expect(rows).toEqual([]);+ expect(hidden).toBe(4);+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);+ });++ test('identity union members\' hidden edges are counted, without fetching them', async () => {+ const { rows, hidden } = await call('get_links', { slug: 'idn/a', source_id: 'default' }, LOCAL);+ expect(rows.map(r => r.context)).toEqual(['a live']);+ expect(hidden).toBe(4);+ expect(hiddenReads().length).toBe(2);+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);+ });++ test('status ended counts the live edges it hides without fetching them', async () => {+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha', status: 'ended' }, REMOTE);+ expect(rows.map(r => r.to_slug)).toEqual(COMPANIES.slice(0, 3));+ expect(hidden).toBe(2);+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);+ });++ test('status all hides nothing and runs no count', async () => {+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha', status: 'all' }, LOCAL);+ expect(rows.length).toBe(5);+ expect(hidden).toBe(0);+ expect(hiddenReads()).toEqual([]);+ });+});