Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(links): count hidden former relationships in SQL instead of fetching them (#7075)

garrytan32257eb5362d
diff --git a/src/core/engine-sql/links.ts b/src/core/engine-sql/links.ts
index b17b2d165f..843aa9b270 100644
--- a/src/core/engine-sql/links.ts
+++ b/src/core/engine-sql/links.ts
@@ -28,10 +28,16 @@ import type { LinkReadScope } from '../link-validity.ts';
/** Bounded link reads annotate rows with relationship status (never filter) when a temporal read is requested. */
function finishTemporal(rows: unknown[], opts?: LinkReadScope): Link[] {
- const t = opts?.temporal;
+ const t = opts?.page?.count ? undefined : opts?.temporal;
return (t ? rows.map(r => annotateTemporalRow(r as Link, t.asOf)) : rows) as Link[];
}
+/** Run one link read; a counting page (#7075) runs it as count(*) and returns one row carrying `link_total`. */
+async function runLinkRead(exec: ScopedRead, opts: LinkReadScope | undefined, statement: SqlFragment): Promise<unknown[]> {
+ if (!opts?.page?.count) return (await exec.run(statement)).rows;
+ return (await exec.run(sqlFragment`SELECT count(*)::int AS link_total FROM (${statement}) counted`)).rows;
+}
+
/** WHERE fragment that keeps only edges matching the caller's temporal policy (walks and fanout). */
function temporalWalkFilter(alias: string, opts?: LinkReadScope) {
return opts?.temporal ? trustedSql(`AND ${relationshipFilterSql(alias, { ...opts.temporal, excludePrivate: opts.excludePrivate })}`) : sqlFragment``;
@@ -355,7 +361,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc
// out of grant. Remote MCP clients always land here.
if (opts?.sourceIds && opts.sourceIds.length > 0) {
const ids = opts.sourceIds;
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -368,7 +374,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc
WHERE f.slug = ${slug} AND f.source_id = ANY(${ids}::text[]) AND t.source_id = ANY(${ids}::text[])
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
// v0.31.8 (D16) + #2200: the federated arm above is the first branch; the
@@ -379,7 +385,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc
// NULL) so links to/from soft-deleted pages stop voting in the graph,
// matching orphans/get/list/search visibility.
if (opts?.sourceId) {
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -392,10 +398,10 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc
WHERE f.slug = ${slug} AND f.source_id = ${opts.sourceId}
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -408,7 +414,7 @@ export async function getLinks(exec: ScopedRead, slug: string, opts?: LinkReadSc
WHERE f.slug = ${slug}
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
@@ -424,7 +430,7 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkRe
// a foreign referrer nor a foreign origin slug is disclosed to the caller.
if (opts?.sourceIds && opts.sourceIds.length > 0) {
const ids = opts.sourceIds;
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -437,13 +443,13 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkRe
WHERE t.slug = ${slug} AND t.source_id = ANY(${ids}::text[]) AND f.source_id = ANY(${ids}::text[])
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
// v0.31.8 (D16) + #2200: federated arm above is first; two below mirror getLinks
// (incl. the #3754 soft-delete endpoint filter on all three arms).
if (opts?.sourceId) {
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -456,10 +462,10 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkRe
WHERE t.slug = ${slug} AND t.source_id = ${opts.sourceId}
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
- const rows = (await exec.run(sqlFragment`
+ const rows = await runLinkRead(exec, opts, sqlFragment`
SELECT f.slug as from_slug, f.source_id as from_source_id,
t.slug as to_slug, t.source_id as to_source_id,
l.link_type, ${page.context}, l.link_source,
@@ -472,7 +478,7 @@ export async function getBacklinks(exec: ScopedRead, slug: string, opts?: LinkRe
WHERE t.slug = ${slug}
AND f.deleted_at IS NULL AND t.deleted_at IS NULL ${trustedSql(privacy)}${page.where}
ORDER BY l.id${page.limit}
- `)).rows;
+ `);
return finish(rows);
}
diff --git a/src/core/link-validity.ts b/src/core/link-validity.ts
index 218c413c3e..20f047e5a6 100644
--- a/src/core/link-validity.ts
+++ b/src/core/link-validity.ts
@@ -398,6 +398,8 @@ export interface LinkReadPage {
relationship?: { policy: EdgeTemporalOpts; keep: 'match' | 'hidden' };
/** Context longer than this is cut, ending in an ellipsis. */
contextChars?: number;
+ /** Count the rows in SQL instead of returning them: the read returns one row whose `link_total` is the count. */
+ count?: boolean;
}
export interface EdgeTemporalOpts {
diff --git a/src/core/ops/links.ts b/src/core/ops/links.ts
index ce5a92e7a2..50c97c1e5b 100644
--- a/src/core/ops/links.ts
+++ b/src/core/ops/links.ts
@@ -351,7 +351,10 @@ async function readLinkEdges(
do {
const { streams, rows } = await read({ ...base, afterId, limit: listing.limit + 1, ...(filtering ? { relationship: { policy: temporal, keep: 'match' as const } } : {}) });
const page = cutLinkPage(streams, rows, listing.limit);
- if (filtering) hidden += (await read({ ...base, afterId, throughId: page.nextId, relationship: { policy: temporal, keep: 'hidden' } })).rows.length;
+ if (filtering) {
+ const counted = await read({ ...(linkType ? { linkType } : {}), ...(fromType ? { fromType } : {}), afterId, throughId: page.nextId, relationship: { policy: temporal, keep: 'hidden' }, count: true });
+ for (const stream of counted.streams) hidden += Number((stream[0] as Link & { link_total?: number } | undefined)?.link_total ?? 0);
+ }
const shaped = filterTemporalLinks(page.rows, temporal);
kept.push(...shaped.kept);
hidden += shaped.hidden;
diff --git a/test/link-hidden-count-7075.test.ts b/test/link-hidden-count-7075.test.ts
new file mode 100644
index 0000000000..9a0cbb60ea
--- /dev/null
+++ b/test/link-hidden-count-7075.test.ts
@@ -0,0 +1,140 @@
+/**
+ * #7075 — get_links / get_backlinks count the former relationships a page's
+ * temporal policy hides with a count(*) in SQL. Regression: the last page of
+ * a listing fetched every hidden edge (context included) only to count them.
+ * The count reported in `gbrain.temporal.hidden` stays the same; no hidden
+ * edge row leaves the database.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { dispatchToolCall } from '../src/mcp/dispatch.ts';
+import { ENTITY_IDENTITY_UNION_CONFIG_KEY, linkEntityIdentity } from '../src/core/entity-identity.ts';
+import type { LinkReadScope } from '../src/core/link-validity.ts';
+import type { Link } from '../src/core/types.ts';
+
+const config = { engine: 'pglite' } as never;
+const LOCAL = { remote: false, sourceId: 'default', config };
+const REMOTE = { remote: true, transport: 'stdio' as const, sourceId: 'default', takesHoldersAllowList: ['world'], config };
+const COMPANIES = ['companies/c0', 'companies/c1', 'companies/c2', 'companies/c3'];
+
+let engine: PGLiteEngine;
+const reads: Array<{ opts?: LinkReadScope; rows: number }> = [];
+
+async function put(slug: string, type = 'note') {
+ await engine.putPage(slug, { type: type as never, title: slug, compiled_truth: `${slug} body`, frontmatter: {} }, { sourceId: 'default' });
+}
+
+async function ended(from: string, to: string) {
+ const r = await dispatchToolCall(engine, 'add_link', { from, to, link_type: 'works_at', valid_from: '2019-01-01', valid_until: '2020-01-01' }, LOCAL as never);
+ if (r.isError) throw new Error(`seed add_link failed: ${r.content[0].text}`);
+}
+
+async function call(op: string, args: Record<string, unknown>, opts: Record<string, unknown>) {
+ reads.length = 0;
+ const r = await dispatchToolCall(engine, op, args, opts as never);
+ if (r.isError) throw new Error(r.content[0].text);
+ const meta = (r as { _meta?: Record<string, unknown> })._meta ?? {};
+ return { rows: JSON.parse(r.content[0].text) as Array<Record<string, unknown>>, hidden: (meta['gbrain.temporal'] as { hidden?: number } | undefined)?.hidden };
+}
+
+const hiddenReads = () => reads.filter(r => r.opts?.page?.relationship?.keep === 'hidden');
+
+beforeAll(async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+ for (const slug of ['hubs/alpha', 'hubs/beta', 'idn/a', 'idn/b', 'targets/t0', 'targets/t1', 'targets/t2']) await put(slug);
+ for (const slug of COMPANIES) await put(slug, 'company');
+ // hubs/alpha: two live edges, then three ended works_at (all on the last page).
+ await engine.addLink('hubs/alpha', 'targets/t0', 'one', 'k000', 'manual');
+ await engine.addLink('hubs/alpha', 'targets/t1', 'two', 'k000', 'manual');
+ for (const c of COMPANIES.slice(0, 3)) await ended('hubs/alpha', c);
+ // hubs/beta: hidden edges between and after live edges, for a paged read.
+ await engine.addLink('hubs/beta', 'targets/t0', 'one', 'k000', 'manual');
+ await ended('hubs/beta', COMPANIES[0]);
+ await engine.addLink('hubs/beta', 'targets/t1', 'two', 'k000', 'manual');
+ await ended('hubs/beta', COMPANIES[1]);
+ await engine.addLink('hubs/beta', 'targets/t2', 'three', 'k000', 'manual');
+ await ended('hubs/beta', COMPANIES[2]);
+ await ended('hubs/beta', COMPANIES[3]);
+ // Identity union: idn/a and idn/b are one entity; each carries hidden edges
+ // (two of them into companies/c0, which the backlinks case also counts).
+ await ended('idn/a', COMPANIES[0]);
+ await ended('idn/b', COMPANIES[0]);
+ await ended('idn/a', COMPANIES[1]);
+ await ended('idn/b', COMPANIES[2]);
+ await engine.addLink('idn/a', 'targets/t0', 'a live', 'k000', 'manual');
+ await engine.setConfig(ENTITY_IDENTITY_UNION_CONFIG_KEY, 'true');
+ await linkEntityIdentity(engine, { entityId: 'thing/idn', slug: 'idn/a', sourceId: 'default' });
+ await linkEntityIdentity(engine, { entityId: 'thing/idn', slug: 'idn/b', sourceId: 'default' });
+
+ const getLinks = engine.getLinks.bind(engine);
+ const getBacklinks = engine.getBacklinks.bind(engine);
+ engine.getLinks = async (slug: string, opts?: LinkReadScope): Promise<Link[]> => {
+ const rows = await getLinks(slug, opts);
+ reads.push({ opts, rows: rows.length });
+ return rows;
+ };
+ engine.getBacklinks = async (slug: string, opts?: LinkReadScope): Promise<Link[]> => {
+ const rows = await getBacklinks(slug, opts);
+ reads.push({ opts, rows: rows.length });
+ return rows;
+ };
+}, 120_000);
+
+afterAll(async () => { if (engine) await engine.disconnect(); }, 60_000);
+
+describe('#7075 hidden-relationship count', () => {
+ test('the last page counts its hidden edges in SQL and reports the same count', async () => {
+ for (const opts of [LOCAL, REMOTE]) {
+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha' }, opts);
+ expect(rows.map(r => r.context)).toEqual(['one', 'two']);
+ expect(hidden).toBe(3);
+ expect(hiddenReads().length).toBeGreaterThan(0);
+ for (const r of hiddenReads()) {
+ expect(r.opts?.page?.count).toBe(true);
+ expect(r.rows).toBe(1);
+ }
+ }
+ });
+
+ test('a paged listing counts each page\'s hidden edges once, in SQL', async () => {
+ const first = await call('get_links', { slug: 'hubs/beta', limit: 2 }, REMOTE);
+ expect(first.rows.map(r => r.context)).toEqual(['one', 'two']);
+ expect(first.hidden).toBe(1);
+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);
+ const second = await call('get_links', { slug: 'hubs/beta', limit: 2, cursor: first.rows[1].next_cursor }, REMOTE);
+ expect(second.rows.map(r => r.context)).toEqual(['three']);
+ expect(second.hidden).toBe(3);
+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);
+ });
+
+ test('get_backlinks counts the same way', async () => {
+ const { rows, hidden } = await call('get_backlinks', { slug: COMPANIES[0] }, REMOTE);
+ expect(rows).toEqual([]);
+ expect(hidden).toBe(4);
+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);
+ });
+
+ test('identity union members\' hidden edges are counted, without fetching them', async () => {
+ const { rows, hidden } = await call('get_links', { slug: 'idn/a', source_id: 'default' }, LOCAL);
+ expect(rows.map(r => r.context)).toEqual(['a live']);
+ expect(hidden).toBe(4);
+ expect(hiddenReads().length).toBe(2);
+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);
+ });
+
+ test('status ended counts the live edges it hides without fetching them', async () => {
+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha', status: 'ended' }, REMOTE);
+ expect(rows.map(r => r.to_slug)).toEqual(COMPANIES.slice(0, 3));
+ expect(hidden).toBe(2);
+ expect(hiddenReads().every(r => r.opts?.page?.count === true && r.rows === 1)).toBe(true);
+ });
+
+ test('status all hides nothing and runs no count', async () => {
+ const { rows, hidden } = await call('get_links', { slug: 'hubs/alpha', status: 'all' }, LOCAL);
+ expect(rows.length).toBe(5);
+ expect(hidden).toBe(0);
+ expect(hiddenReads()).toEqual([]);
+ });
+});