Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Eval capture keeps fact results, and a re-created source keeps its rows

garrytan18044e51462b
diff --git a/src/core/feedback/store.ts b/src/core/feedback/store.ts
index d7efb947ff..aeab0c2a9c 100644
--- a/src/core/feedback/store.ts
+++ b/src/core/feedback/store.ts
@@ -334,7 +334,8 @@ export async function scrubPurgedSlugs(
* #9068: insert one captured eval candidate with only the slugs a page of its sources (any source when it names none)
* still holds, in retrieval order. Each probe takes the feedback writers' key-share lock on the page row, so a capture
* deferred past its query's response either commits before a purge takes the row, and the purge's scrub strips it, or
- * waits for the purge and drops the slug. Both engines run this statement.
+ * waits for the purge and drops the slug. A facts-arm result (`facts/<id>`, facts-arm.ts) names a fact, not a page:
+ * it stays while that fact row does. Both engines run this statement.
*/
export async function insertEvalCandidate(engine: Pick<BrainEngine, 'executeRaw'>, input: EvalCandidateInput): Promise<number> {
const rows = await engine.executeRaw<{ id: number | string }>(
@@ -346,6 +347,8 @@ export async function insertEvalCandidate(engine: Pick<BrainEngine, 'executeRaw'
ARRAY(SELECT u.s FROM unnest($3::text[]) WITH ORDINALITY AS u(s, n)
WHERE EXISTS (SELECT 1 FROM pages pg WHERE pg.slug = u.s
AND (cardinality($5::text[]) = 0 OR pg.source_id = ANY($5::text[])) FOR KEY SHARE)
+ OR CASE WHEN u.s ~ '^facts/[0-9]{1,18}$' THEN EXISTS (SELECT 1 FROM facts f WHERE f.id = substr(u.s, 7)::bigint
+ AND (cardinality($5::text[]) = 0 OR f.source_id = ANY($5::text[]))) ELSE false END
ORDER BY u.n),
$4::int[], $5::text[], $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING id`,
diff --git a/src/core/source-delete.ts b/src/core/source-delete.ts
index 20e398411b..e7d18647ba 100644
--- a/src/core/source-delete.ts
+++ b/src/core/source-delete.ts
@@ -19,7 +19,9 @@ export const unverifiedArchiveReason = (id: string) =>
* claimed. A binding of another incarnation is never touched here; `gbrain
* repair orphan-bindings` removes those. The same statement runs
* removedSourceScrubCtes, and a second statement repeats it once the row is
- * gone. Returns whether a row was deleted.
+ * gone, unless a source with this id exists again by then: a replacement
+ * created between the two statements keeps its rows. Returns whether a row
+ * was deleted.
*/
export async function deleteSourceRow(
engine: Pick<BrainEngine, 'executeRaw'>,
@@ -38,7 +40,7 @@ export async function deleteSourceRow(
[id],
);
if (!rows.length) return false;
- await engine.executeRaw(`WITH ${removedSourceScrubCtes('true')} SELECT 1`, [id]);
+ await engine.executeRaw(`WITH ${removedSourceScrubCtes('NOT EXISTS (SELECT 1 FROM sources WHERE id = $1)')} SELECT 1`, [id]);
return true;
}
@@ -48,8 +50,8 @@ export async function deleteSourceRow(
* context_volunteer_events rows, and each eval candidate's captured slugs that no other source the candidate covers
* still has a page for (retrieval_weights and mention_gazetteer_entries cascade from sources). Every removal runs
* them in the statement that deletes the sources row (gated on that delete), so the source and those rows go
- * together, then again, ungated, in a later statement: that statement reads past a feedback write or eval capture
- * that held one of the source's page rows while the delete waited for it (a later one finds no page).
+ * together, then again in a later statement: that statement reads past a feedback write or eval capture that held one
+ * of the source's page rows while the delete waited for it (a later one finds no page).
*/
export function removedSourceScrubCtes(gate: string): string {
const deletes = ['retrieval_event_pages', 'retrieval_event_links', 'retrieval_feedback', 'context_volunteer_events']
diff --git a/test/purge-scrub-followups-9068.test.ts b/test/purge-scrub-followups-9068.test.ts
index 317814ba75..e5bf5e4ef8 100644
--- a/test/purge-scrub-followups-9068.test.ts
+++ b/test/purge-scrub-followups-9068.test.ts
@@ -8,10 +8,12 @@
* retrieval logs, ratings, weights, the volunteer log, the gazetteer and the eval candidates' captured slugs for
* that source in the delete's transaction, and so does the transcript stale-part delete, which now calls it;
* (c) removing a source (unmanaged removeSource -> deleteSourceRow, and the managed lifecycle remove) drops every
- * such row of the source, and strips its slugs from eval candidates no other covered source still answers.
- * 2. Fails when: logEvalCandidate inserts slugs without probing the page row under a key-share lock, a hard delete or a
- * source removal leaves any of those rows behind, or a scrub reaches another source's rows or a slug another
- * source the candidate covers still holds.
+ * such row of the source, and strips its slugs from eval candidates no other covered source still answers;
+ * (d) a capture keeps a facts-arm result (facts/<id>) while its fact exists; (e) a source re-created with the same
+ * id between deleteSourceRow's delete and its second scrub keeps its rows.
+ * 2. Fails when: logEvalCandidate inserts slugs without probing the page row under a key-share lock or drops a live
+ * fact's result, a hard delete or a source removal leaves any of those rows behind, or a scrub reaches another
+ * source's rows, a re-created source's rows or a slug another source the candidate covers still holds.
* 3. Existing coverage: purge-retrieval-feedback-8821 and purge-name-stores-9068 pin delete_page purge and the
* tombstone purge only; nothing pinned the capture, the connector and stale-part deletes, or source removal.
* 4. Real seams: PGLite, and Postgres when DATABASE_URL is set (test/postgres-unit-arms.txt); the feedback store's
@@ -35,6 +37,7 @@ import { captureEvalCandidate } from '../src/core/eval-capture.ts';
import { withResponseDeferral } from '../src/core/background-work.ts';
import { deleteConnectorPage } from '../src/core/connectors/api-source.ts';
import { removeSource } from '../src/core/sources-ops.ts';
+import { deleteSourceRow } from '../src/core/source-delete.ts';
import { runTranscriptsIngest } from '../src/core/transcripts/ingest.ts';
import { buildTranscriptSlug } from '../src/core/transcripts/types.ts';
import { registerLocalWriter } from '../src/core/persistence/identity.ts';
@@ -145,6 +148,18 @@ describe('#9068 an eval capture does not write a purged slug back', () => {
}
}, 120_000);
+ test('a capture keeps a facts-arm result while its fact exists', async () => {
+ for (const engine of engines) {
+ await page(engine, 'default', 'notes/live-page');
+ const [fact] = await engine.executeRaw<{ id: number | string }>(
+ `INSERT INTO facts (fact, source, source_id) VALUES ('alice-example joined acme-example', 'fixture', 'default') RETURNING id`);
+ const query = `facts arm ${randomUUID()}`;
+ await engine.logEvalCandidate(candidate(query, [`facts/${fact!.id}`, 'facts/999999999', 'notes/live-page'], ['default']));
+ expect({ engine: engine.kind, slugs: await capturedSlugs(engine, query) })
+ .toEqual({ engine: engine.kind, slugs: [[`facts/${fact!.id}`, 'notes/live-page']] });
+ }
+ }, 120_000);
+
const postgresTest = testBackends().includes('postgres') ? test : test.skip;
postgresTest('on Postgres a capture during an open purge waits for it and drops the slug', async () => {
const engine = engines.find(e => e.kind === 'postgres')!;
@@ -240,6 +255,25 @@ describe('#9068 removing a source scrubs every row that names its pages', () =>
}
}), 120_000);
+ test('a source re-created between the delete and its second scrub keeps its rows', () => withEnv({ GBRAIN_HOME: home }, async () => {
+ for (const engine of engines) {
+ const id = 'source-reused', slug = 'notes/reused-page', query = `reused ${randomUUID()}`;
+ await traced(engine, id, slug);
+ let calls = 0;
+ const racing = { executeRaw: async (sql: string, params?: unknown[]) => {
+ if (calls++ === 1) {
+ await traced(engine, id, slug);
+ await engine.logEvalCandidate(candidate(query, [slug, OTHER], [id]));
+ }
+ return engine.executeRaw(sql, params);
+ } } as Pick<BrainEngine, 'executeRaw'>;
+ expect(await deleteSourceRow(racing, id)).toBe(true);
+ expect({ engine: engine.kind, residue: await residue(engine, id, slug) }).toEqual({ engine: engine.kind, residue: { retrieval_event_pages: 1,
+ retrieval_event_links: 1, retrieval_feedback: 2, retrieval_weights: 2, context_volunteer_events: 1, mention_gazetteer_entries: 1 } });
+ expect(await capturedSlugs(engine, query)).toEqual([[slug, OTHER]]);
+ }
+ }), 120_000);
+
test('the managed lifecycle remove does the same', async () => {
for (const engine of engines) {
const lifecycleHome = mkdtempSync(join(tmpdir(), 'gbrain-purge-followups-managed-'));