Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Refuse re-created links that name a page the writer can't read, and keep dates written during the v297 index swap (GST-2228)

garrytan613d606cfaf2
diff --git a/src/core/deletion-inventory.ts b/src/core/deletion-inventory.ts
index 9a28847f38..7edd63263e 100644
--- a/src/core/deletion-inventory.ts
+++ b/src/core/deletion-inventory.ts
@@ -99,7 +99,7 @@ const ROWS: readonly Row[] = [
['ingest_log', 'out_of_scope', 'probed_reported', 'source_id source_type source_ref pages_updated summary writer_version', 'summary pages_updated'],
['link_edge_proposals', 'out_of_scope', 'probed_reported', 'source_id link_type evidence_hash status model generated_line detail', 'generated_line detail'],
['link_relationships', 'out_of_scope', 'no_memory_text', 'link_type scope source_id semantics status_now evidence_hash'],
- ['link_transitions', 'out_of_scope', 'no_memory_text', 'source_id link_type kind date_precision producer line_hash'],
+ ['link_transitions', 'out_of_scope', 'no_memory_text', 'source_id link_type kind date_precision producer line_hash link_source'],
['links', 'out_of_scope', 'probed_reported', 'link_type context link_source link_kind origin_field resolution_type assertion_tense', 'context'],
['loop_suppressions', 'out_of_scope', 'no_memory_text', 'source_id kind value'],
['mcp_request_log', 'out_of_scope', 'probed_reported', 'token_name agent_name operation status params error_message', 'params error_message'],
diff --git a/src/core/links-fence-projection.ts b/src/core/links-fence-projection.ts
index 492b1754af..b4ee04231f 100644
--- a/src/core/links-fence-projection.ts
+++ b/src/core/links-fence-projection.ts
@@ -7,7 +7,9 @@
* source of the row whose cell stated it (link_source), and transitions are
* compared per row, so a row whose dates changed replaces only its own
* statements; one with no source (written before the column existed) is
- * replaced by the row that states the same date. Database-only manual edges
+ * replaced by the row that states the same date. While migration v297 has added the column but not yet swapped in the
+ * identity index that includes it, statements are written without a source, as before the migration, so a second
+ * row's same-date statement is not dropped in place of a stored one. Database-only manual edges
* (origin_field NULL) and every derived producer are left alone. A row whose
* target is not a live page in the source, or whose type the active schema
* pack does not declare, yields no edge and stays in the file.
@@ -18,6 +20,7 @@ import { sanitizeForJsonb } from './batch-rows.ts';
import { refreshRelationships, type RelationshipKey } from './link-relationships.ts';
import { loadActivePackForLocalEngine } from './schema-pack/best-effort.ts';
import { LINKS_FENCE_ORIGIN_FIELD, readPageLinksFence, type LinksFenceRow } from './links-fence.ts';
+import { identityIndexHasSource } from './schema-migrations/v297-link-transitions-source.ts';
type Tx = Pick<BrainEngine, 'executeRaw'>;
@@ -62,10 +65,14 @@ export async function projectLinksFence(tx: Tx, page: { id: number | string; sou
});
const transitionKey = (row: { to_page_id: number; link_type: string; link_source: string | null; kind: string; occurred_on: string }) =>
`${Number(row.to_page_id)}\u0000${row.link_type}\u0000${row.link_source ?? '\u0001'}\u0000${row.kind}\u0000${row.occurred_on}`;
- const transitions = new Map(desired.flatMap(row => [
- ...(row.validFrom ? [{ to_page_id: row.to_page_id, link_type: row.link_type, link_source: row.link_source, kind: 'start', occurred_on: row.validFrom }] : []),
- ...(row.validUntil ? [{ to_page_id: row.to_page_id, link_type: row.link_type, link_source: row.link_source, kind: 'end', occurred_on: row.validUntil }] : []),
- ]).map(row => [transitionKey(row), row]));
+ const tagged = (existingTransitions.length > 0 || desired.some(row => row.validFrom || row.validUntil)) && await identityIndexHasSource(tx);
+ const transitions = new Map(desired.flatMap(row => {
+ const link_source = tagged ? row.link_source : null;
+ return [
+ ...(row.validFrom ? [{ to_page_id: row.to_page_id, link_type: row.link_type, link_source, kind: 'start', occurred_on: row.validFrom }] : []),
+ ...(row.validUntil ? [{ to_page_id: row.to_page_id, link_type: row.link_type, link_source, kind: 'end', occurred_on: row.validUntil }] : []),
+ ];
+ }).map(row => [transitionKey(row), row]));
const haveTransitions = new Set(existingTransitions.map(transitionKey));
const staleTransitions = existingTransitions.filter(row => !transitions.has(transitionKey(row)));
const newTransitions = [...transitions.entries()].filter(([key]) => !haveTransitions.has(key)).map(([, row]) => row);
diff --git a/src/core/persistence/manual-links.ts b/src/core/persistence/manual-links.ts
index 527f14ebc6..42964b6d61 100644
--- a/src/core/persistence/manual-links.ts
+++ b/src/core/persistence/manual-links.ts
@@ -13,6 +13,7 @@ import { linksFenceProjectionRows, projectLinksFence } from '../links-fence-proj
import { removeManualTransitions } from '../link-temporal-apply.ts';
import { refreshRelationships, type RelationshipKey } from '../link-relationships.ts';
import { executeRawJsonb } from '../sql-query.ts';
+import { authorizePageVisibility } from './page-visibility.ts';
const optional = (value: unknown) => typeof value === 'string' && value ? value : undefined;
@@ -96,7 +97,8 @@ async function assertRemovalAttributable(engine: BrainEngine, pageId: number, re
* The publication projects the fence (canonical-projections.ts) and the apply projects it once more, so a fence
* the write left unchanged (an idempotent add) still has its edges. `add_link` drops the database-only twin of
* the edge it fences (same endpoints, type and provenance, no origin); on a page with no links fence, the section it
- * creates lists the edges a dropped section left first, each with its own dates (`keptLinksFenceRows`). `remove_link`
+ * creates lists the edges a dropped section left first, each with its own dates (`keptLinksFenceRows`), and a remote
+ * writer that cannot read one of their targets gets the same page_not_found as for a missing page. `remove_link`
* deletes the removed rows' own dated statements, whatever their source, and also the matching database-only edges
* written before the fence existed, with their manual dated statements, so a removal only moves the database toward
* git; edges a reconciliation producer derives from the page text stay, and the result names them
@@ -114,6 +116,7 @@ export async function prepareManualLinkMutation(engine: BrainEngine, row: WriteR
const pageId = Number(stored.page.id);
const absent = readPageLinksFence(snapshot.page).state === 'absent';
const kept = row.operation === 'add_link' && absent ? await keptLinksFenceRows(engine, pageId) : [];
+ for (const keptRow of kept) await authorizePageVisibility(engine, row.authority, keptRow.to);
if (row.operation === 'remove_link' && absent) await assertRemovalAttributable(engine, pageId, { to, linkType, linkSource });
const current = kept.length ? replaceOrInsertLinksFence(snapshot.page[section] ?? '', renderLinksFence(kept)) : snapshot.page[section] ?? '';
let body: string;
diff --git a/src/core/schema-migrations/v297-link-transitions-source.ts b/src/core/schema-migrations/v297-link-transitions-source.ts
index 4f0f4ab9f4..236fb0e957 100644
--- a/src/core/schema-migrations/v297-link-transitions-source.ts
+++ b/src/core/schema-migrations/v297-link-transitions-source.ts
@@ -43,7 +43,7 @@ const POSTGRES_15_GATE = `DO $$ BEGIN
END $$`;
/** Whether the canonical identity index already includes link_source (fresh install, or a finished swap). */
-export async function identityIndexHasSource(engine: BrainEngine): Promise<boolean> {
+export async function identityIndexHasSource(engine: Pick<BrainEngine, 'executeRaw'>): Promise<boolean> {
const [row] = await engine.executeRaw<{ has: boolean }>(
`SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indexrelid = to_regclass($1) AND a.attname = 'link_source') AS has`, [LINK_TRANSITIONS_IDENTITY_INDEX]);
diff --git a/test/link-dates-per-row-gst2228.test.ts b/test/link-dates-per-row-gst2228.test.ts
index 61f4702e79..cccc536802 100644
--- a/test/link-dates-per-row-gst2228.test.ts
+++ b/test/link-dates-per-row-gst2228.test.ts
@@ -131,6 +131,39 @@ for (const backend of testBackends()) {
expect({ revision: await revision(engine), statements: await statements(engine), edges: await edges(engine) }).toEqual(before);
}, opts), 180_000);
+ test('statements from before the source column that do not fit the one kept row: add_link refuses dates_do_not_fit and changes nothing', () => managedBrain(async ({ engine, ctx }) => {
+ await seed(ctx);
+ await put(ctx, ALICE, PERSON(`Alice v1.\n\n${SECTION([FIRST_STINT])}`));
+ await put(ctx, ALICE, PERSON(DROPPED));
+ await forgetSources(engine);
+ await engine.executeRaw(`INSERT INTO link_transitions (source_id, from_page_id, to_page_id, link_type, kind, occurred_on, producer, origin_page_id)
+ SELECT 'default', f.id, t.id, 'works_at', 'start', '2023-01-01', 'manual', f.id FROM pages f, pages t
+ WHERE f.source_id='default' AND f.slug=$1 AND t.source_id='default' AND t.slug=$2`, [ALICE, ACME]);
+ const before = { revision: await revision(engine), statements: await statements(engine), edges: await edges(engine) };
+ expect(before.statements).toEqual([`${ACME} end 2021-01-01 NULL`, `${ACME} start 2020-01-01 NULL`, `${ACME} start 2023-01-01 NULL`]);
+ await expect(addBeta(ctx)).rejects.toMatchObject({ code: 'link_dates_unattributed', message: expect.stringContaining('do not fit its one row') });
+ expect({ revision: await revision(engine), statements: await statements(engine), edges: await edges(engine) }).toEqual(before);
+ }, opts), 180_000);
+
+ test('a remote writer that cannot read a kept row\'s target is refused as for a missing page, and the section is not rebuilt', () => managedBrain(async ({ engine, ctx }) => {
+ const hidden: LinksFenceRow = { to: 'companies/hidden-example', type: 'works_at', source: 'manual', validFrom: '2020-01-01', context: 'private stint' };
+ await seed(ctx);
+ await put(ctx, hidden.to, '---\ntype: company\ntitle: Hidden Example\nvisibility: private\n---\n\n# Hidden Example\n\nA private company.\n');
+ await put(ctx, ALICE, PERSON(`Alice v1.\n\n${SECTION([hidden])}`));
+ await put(ctx, ALICE, PERSON(DROPPED));
+ const clientId = `client-${randomUUID().slice(0, 8)}`;
+ await engine.executeRaw(`INSERT INTO oauth_clients(client_id,client_secret_hash,client_name,scope,source_id) VALUES($1,'fixture-hash','example-client','read write','default')`, [clientId]);
+ const remote = { ...ctx, remote: true, transport: 'http', auth: { token: 'fixture', clientId, principal: { kind: 'oauth_client', id: clientId }, scopes: ['read', 'write'], sourceId: 'default' } } as OperationContext;
+ const before = await revision(engine);
+ const missing = await op(remote, 'add_link', { from: ALICE, to: hidden.to, link_type: BETA.type }).then(() => null, (error: { code?: string; message?: string }) => ({ code: error.code, message: error.message }));
+ expect(missing?.code).toBe('page_not_found');
+ await expect(op(remote, 'add_link', { from: ALICE, to: BETA.to, link_type: BETA.type, context: BETA.context })).rejects.toMatchObject(missing!);
+ expect(await revision(engine)).toBe(before);
+ expect(await pageRows(engine)).toEqual([]);
+ await addBeta(ctx);
+ expect(await pageRows(engine)).toEqual(sortRows([hidden, BETA]));
+ }, opts), 180_000);
+
test('statements from before the source column, one kept row: add_link puts them on that row and labels them with its source', () => managedBrain(async ({ engine, ctx, root }) => {
await seed(ctx);
await put(ctx, ALICE, PERSON(`Alice v1.\n\n${SECTION([FIRST_STINT])}`));
diff --git a/test/link-transitions-source-migration-gst2228.test.ts b/test/link-transitions-source-migration-gst2228.test.ts
index e294398f0d..02d19dcf76 100644
--- a/test/link-transitions-source-migration-gst2228.test.ts
+++ b/test/link-transitions-source-migration-gst2228.test.ts
@@ -6,9 +6,9 @@
* name (CONCURRENTLY on Postgres), and one short transaction swaps it into the canonical name. A rerun after the run
* stopped at any step finishes the job, and an INVALID leftover of an interrupted concurrent build is dropped and
* rebuilt. Statements stored before the column keep a NULL source and are not rewritten.
- * 2. Writes between the column and the swap: the old index still keys a statement without its source, so a second row's
- * statement for the same relationship and date is dropped by ON CONFLICT DO NOTHING, as before the migration; the
- * page's next write after the swap adds it.
+ * 2. Writes between the column and the swap: the old index still keys a statement without its source, so the projection
+ * writes statements without a source there, as before the migration, and never deletes a stored statement for a
+ * replacement the old index would drop; the page's next write after the swap labels them per row.
* 3. Real seams: runMigrations and buildIndexOnline on PGLite, and on Postgres when DATABASE_URL is set (testBackends).
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
@@ -136,18 +136,18 @@ describe('GST-2228 v297 link_transitions_source', () => {
});
}
- test(`[${backend}] a links-section write between the column and the swap keeps one statement per date, and the next write after the swap adds the other row's`, async () => {
+ test(`[${backend}] a links-section write between the column and the swap keeps one sourceless statement per date, and the next write after the swap labels each row's`, async () => {
const { engine, alice } = brain();
const rows = [
{ to: 'companies/acme-example', type: 'works_at', source: 'manual', context: 'typed in', validFrom: '2020-01-01' },
{ to: 'companies/acme-example', type: 'works_at', source: 'crm-import', context: 'from the import', validFrom: '2020-01-01' },
];
- const statements = async () => (await engine.executeRaw<{ source: string }>(
+ const statements = async () => (await engine.executeRaw<{ source: string | null }>(
"SELECT link_source AS source FROM link_transitions WHERE origin_page_id=$1 AND producer='manual' ORDER BY 1", [alice])).map(r => r.source);
await beforeV297();
await engine.executeRaw('ALTER TABLE link_transitions ADD COLUMN link_source TEXT');
await projectLinksFence(engine, { id: alice, source_id: 'default' }, rows);
- expect(await statements()).toEqual(['manual']);
+ expect(await statements()).toEqual([null]);
await runMigrations(engine);
expect(await identity()).toEqual(DONE);
await projectLinksFence(engine, { id: alice, source_id: 'default' }, rows);