Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Keep a private origin until every derived page it hides is labelled, including held tombstones and origins near the cutoff
garrytana77e769e610d
diff --git a/src/commands/doctor/checks/purge-parks.ts b/src/commands/doctor/checks/purge-parks.tsindex cbccd81806..2f95bce236 100644--- a/src/commands/doctor/checks/purge-parks.ts+++ b/src/commands/doctor/checks/purge-parks.ts@@ -65,7 +65,7 @@ export async function checkDerivedOriginParked(engine: BrainEngine): Promise<Chereturn {name: ORIGIN_NAME, status: 'warn', details: { parked: parks.length, truncated: parks.length > origins.length, origins },message: `${parks.length} soft-deleted page(s) are kept from the purge, because a page derived from each reads as private only through it and could not be labelled private: ${listed}`- + `${parks.length > origins.length ? ` (first ${origins.length} of ${parks.length} shown)` : ''}. Fix the cause each reason names, or give the blocking page its own visibility label; the next purge labels the rest and purges the origin.`,+ + `${parks.length > origins.length ? ` (first ${origins.length} of ${parks.length} shown)` : ''}. Fix the cause each reason names, or give the blocking page its own visibility label; the next purge labels the rest and frees the origin, and the purge after removes it (a consented gbrain pages purge-deleted does both).`,...(first ? { fix: agentFix(['gbrain', 'repair', 'visibility', '--source', first.source_id, '--slug', first.blocking_slug!],'Previews giving the blocking page its own visibility label; nothing is written.', ORIGIN_NAME) } : {}),};diff --git a/src/commands/pages.ts b/src/commands/pages.tsindex dbbbb17e19..dba0e03d4a 100644--- a/src/commands/pages.ts+++ b/src/commands/pages.ts@@ -47,6 +47,7 @@ async function runPurgeDeleted(engine: BrainEngine, args: string[]): Promise<voiconst { olderThanHours, dryRun, json, retryParked } = parsePurgeArgs(args);// #6741: one selector for the preview, the consent and the run (selectPurgeCandidates): the same cutoff, rename// holds, writer state and parks; --retry-parked puts the parked pages the cutoff selects back among the candidates.+ if (!dryRun) await (await import('../core/repair/origin-carry.ts')).retryParkedOrigins(engine, olderThanHours);const selection = await selectPurgeCandidates(engine, olderThanHours, { retryParked });const retried = selection.parked.filter(park => park.retried).length;const notes = { ...(selection.parked.length ? { parked: selection.parked, retry_parked: retryParked } : {}), ...(selection.blocked.length ? { blocked: selection.blocked } : {}) };diff --git a/src/core/persistence/purge-deleted.ts b/src/core/persistence/purge-deleted.tsindex 9c4b0e3e3d..39c4a8045a 100644--- a/src/core/persistence/purge-deleted.ts+++ b/src/core/persistence/purge-deleted.ts@@ -14,7 +14,7 @@ import { linkStreamSlugs } from './page-purge.ts';import { identityDowngradePurgeHold } from '../sync-identity-downgrades.ts';import { atomRetirementPurgeHold } from './atom-retirement-hold.ts';import { derivedRetirementPurgeHold } from './derived-retirement-hold.ts';-import { DERIVED_ORIGIN_PARKED_CODE, derivedOriginPurgeHold, type DerivedOriginPark } from './derived-origin-hold.ts';+import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, derivedOriginPurgeHold, type DerivedOriginPark } from './derived-origin-hold.ts';import { writerOnboardingPreflight } from './onboarding.ts';import { clearPurgePark, isParked, notePurgeSourceWriter, PURGE_PARK_LIMIT, purgeParkKey, prunePurgeParks, readPurgeParks, recordPurgeFailure, type PurgePark } from './purge-parks.ts';import { ItemRun, type ItemFailure } from '../item-failure.ts';@@ -90,6 +90,7 @@ export function purgeConsentRequest(count: number, olderThanHours: number, json:/** The `purge_deleted_pages` op (`gbrain call`): the D6 consent first (`yes` is `--yes`); an empty purge set needs none. */export async function consentedPurgeDeletedPages(engine: BrainEngine, olderThanHours: number, yes: boolean) {+ await (await import('../repair/origin-carry.ts')).retryParkedOrigins(engine, olderThanHours);const preview = await selectPurgeCandidates(engine, olderThanHours);if (preview.count === 0) return { status: 'purged', count: 0, slugs: [] };await requireConsent(purgeConsentRequest(preview.count, olderThanHours, false, yes ? ['--yes'] : [], false, 0, preview.sync_unreviewed));@@ -331,8 +332,10 @@ export function purgePhaseNote(result: DeletedPagePurge): string {export type PurgeFilter = (page: { source_id: string; slug: string }) => boolean;export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter } = {}): Promise<DeletedPagePurge> {- const origins = originParked(await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours));+ const carried = await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours);+ const origins = originParked(carried.parks);const result = await purgeExpired(engine, olderThanHours, opts);+ await clearDerivedOriginParks(engine, carried.unparked);return origins.length ? { ...result, blocked: [...origins, ...result.blocked], origin_parked: origins.length } : result;}diff --git a/src/core/repair/origin-carry.ts b/src/core/repair/origin-carry.tsindex 171ffdc072..510c3d56b3 100644--- a/src/core/repair/origin-carry.ts+++ b/src/core/repair/origin-carry.ts@@ -20,7 +20,7 @@ import { serializePageToMarkdown } from '../markdown.ts';import { isSegmentedPage } from '../page-body/page-shape.ts';import type { LabelEdit } from '../persistence/label-edit.ts';import { maintenancePreflight, publishMaintenancePages } from '../persistence/prepared-maintenance.ts';-import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, parkDerivedOrigin, pruneDerivedOriginParks, type DerivedOriginPark } from '../persistence/derived-origin-hold.ts';+import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, parkDerivedOrigin, pruneDerivedOriginParks, readDerivedOriginParks, type DerivedOriginPark } from '../persistence/derived-origin-hold.ts';import { privacyPartSql, readHonorAliases, withoutCallerAccess } from '../search/private-visibility.ts';import { filesInCheckout, importAtRevision, writeLabelFile } from './label-write.ts';import { dependentsSql } from './private-labels.ts';@@ -152,26 +152,30 @@ export async function deletePageCarryingPrivate(ctx: OperationContext, p: Recordif (origin && result.status === 'purged') await clearDerivedOriginParks(ctx.engine, [origin.id]);return result;}+ await parkDerivedOrigin(ctx.engine, parked(origin, block));const result = origin.deleted ? { status: 'held', slug: origin.slug, source_id: sourceId, request_id: p.request_id }: await submitPageMutation(ctx, { operation: 'delete_page', params: { ...p, purge: false, request_id: derivedRequestId(String(p.request_id)) } });- await parkDerivedOrigin(ctx.engine, parked(origin, block));return { ...result, purge_held: { code: DERIVED_ORIGIN_PARKED_CODE,message: `Not purged: ${block.slug ?? 'a page derived from it'} is private only through ${origin.slug} and could not be labelled private (${block.code ?? 'error'}: ${block.reason}). `+ `${origin.slug} is soft-deleted and parked: no purge removes it until that page carries its own label; gbrain doctor names it as derived_origin_parked.` } };}+/** Hours before its cutoff a tombstone is carried, so none crosses the cutoff while the carry runs and is purged uncarried. */+export const CARRY_LEAD_HOURS = 24;+/**- * Before a purge: the expired soft-deleted origins get their dependents labelled private, judged against every expired- * origin at once (a page private through two of them opens only when both go); each one still blocked is parked (the- * purge skips it) and the rest lose any earlier park. The parks written this run.+ * Before a purge: the soft-deleted origins expired or expiring within CARRY_LEAD_HOURS get their dependents labelled+ * private, judged against all of them at once (a page private through two of them opens only when both go); each one+ * still blocked is parked (the purge skips it). The parks written this run, and the origins whose earlier park no longer+ * applies: the caller clears those after its purge, so an origin parked when the purge was previewed stays this run.*/-export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHours: number): Promise<DerivedOriginPark[]> {+export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHours: number): Promise<{ parks: DerivedOriginPark[]; unparked: number[] }> {await pruneDerivedOriginParks(engine);const origins = await withoutCallerAccess(async () => {await readHonorAliases(engine);return engine.executeRaw<{ id: number; source_id: string; slug: string }>(`SELECT p.id, p.source_id, p.slug FROM pages pWHERE p.deleted_at IS NOT NULL AND p.deleted_at < now() - ($1 || ' hours')::interval AND ${originSql('p')} ORDER BY p.id`,- [String(Math.max(0, Math.floor(olderThanHours)))], bounded);+ [String(Math.max(0, Math.floor(olderThanHours) - CARRY_LEAD_HOURS))], bounded);});const parks: DerivedOriginPark[] = [];const batch = `derived-origin-carry:purge-${randomUUID()}`;@@ -183,6 +187,14 @@ export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHourawait parkDerivedOrigin(engine, parked(origin, block));parks.push({ ...parked(origin, block), parked_at: new Date().toISOString() });}- await clearDerivedOriginParks(engine, expired.filter(origin => !blocked.has(origin.id)).map(origin => origin.id));- return parks;+ return { parks, unparked: expired.filter(origin => !blocked.has(origin.id)).map(origin => origin.id) };+}++/**+ * A consented manual purge first retries the label for every parked origin and unparks the ones it frees, so its preview+ * and consent list them (otherwise a brain whose only expired pages are parked origins never gets past "nothing to purge").+ */+export async function retryParkedOrigins(engine: BrainEngine, olderThanHours: number): Promise<void> {+ if (!(await readDerivedOriginParks(engine)).length) return;+ await clearDerivedOriginParks(engine, (await carryPrivateBeforePurge(engine, olderThanHours)).unparked);}diff --git a/src/core/repair/private-labels.ts b/src/core/repair/private-labels.tsindex b4de44fa36..e7ff4c9f75 100644--- a/src/core/repair/private-labels.ts+++ b/src/core/repair/private-labels.ts@@ -238,7 +238,8 @@ async function planRestore(engine: BrainEngine, scope: RepairScope, selection: S* them go too), `keys` is unused, an atom also ties to the page a rename moved its `source_slug` to (slug_aliases and* slug_rename_history, read before a purge scrubs them). A page that names readers is judged by the same privacy rules,* since readable_by never relaxes them: it is returned when the removal would open it to its listed readers. A soft-deleted- * page is a candidate too, since a restore brings it back with the origin row gone.+ * page is a candidate too, since a restore brings it back with the origin row gone, and so is a removed page tied to+ * another removed page (judged on its own row, against the postimage), since another hold may keep it from the purge.*/export function dependentsSql(mode: 'clear' | 'remove' = 'clear'): string {const remove = mode === 'remove';@@ -265,12 +266,13 @@ export function dependentsSql(mode: 'clear' | 'remove' = 'clear'): string {UNION ALL SELECT r.id, h.cleared_id FROM hits h JOIN refs r ON r.source_id = h.source_id AND r.kind = 'depth' AND r.ref = h.slugWHERE h.kind = 'declared'${live(' AND r.deleted_at IS NULL')}UNION ALL SELECT p.id, t.cleared_id FROM provenance t JOIN pages p ON p.id = t.from_page_id AND p.source_id = t.source_id${live(' AND p.deleted_at IS NULL')}),- candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)),+ candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE ${remove ? 't.id <> t.cleared_id' : 'NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)'}),${remove ? `+ candidate_rows AS MATERIALIZED (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM pages a WHERE a.id IN (SELECT id FROM candidates)),` : ''}public_after AS MATERIALIZED (WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatterFROM pages pg LEFT JOIN cleared c ON c.id = pg.id${remove ? ' WHERE c.id IS NULL' : ''})- SELECT after_clear.id FROM (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k- JOIN pages a ON a.id = k.id OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})+ SELECT after_clear.id FROM (${remove ? 'SELECT * FROM candidate_rows' : `SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k+ JOIN pages a ON a.id = k.id`} OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS categoryFROM candidates cand JOIN pages p ON p.id = cand.iddiff --git a/test/derived-origin-carry-private.test.ts b/test/derived-origin-carry-private.test.tsindex 9faf57efb6..d3e6b48805 100644--- a/test/derived-origin-carry-private.test.ts+++ b/test/derived-origin-carry-private.test.ts@@ -33,7 +33,7 @@ import { dispatchToolCall } from '../src/mcp/dispatch.ts';import { __resetPrivateVisibilityCacheForTests } from '../src/core/search/private-visibility.ts';import { setOwner } from '../src/core/access/owner.ts';import { bindCredential } from '../src/core/access/members.ts';-import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';+import { consentedPurgeDeletedPages, purgeDeletedPagesCoordinated, selectPurgeCandidates } from '../src/core/persistence/purge-deleted.ts';import { checkDerivedOriginParked } from '../src/commands/doctor/checks/purge-parks.ts';import type { AuthInfo } from '../src/core/ops/contract.ts';import { testBackends } from './helpers/test-backends.ts';@@ -270,10 +270,11 @@ for (const kind of testBackends()) {.toContainEqual(expect.objectContaining({ slug: origin, blocking_slug: digest }));const restored = await localCall('restore_page', { slug: digest, request_id: randomUUID(), expected_revision: await revision(digest) });expect(restored.isError ? restored.text : '').toBe('');+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);+ expect((await fm(digest)).visibility).toBe('private');const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });expect(result.slugs).toContain(origin);await nothingKept(origin);- expect((await fm(digest)).visibility).toBe('private');__resetPrivateVisibilityCacheForTests();for (const who of ['unbound', 'alice'] as const) expect({ who, seen: await seen(who, k, 'digest') }).toEqual({ who, seen: none });expect({ owner: await seen('owner', k, 'digest') }).toEqual({ owner: every });@@ -295,7 +296,67 @@ for (const kind of testBackends()) {for (const who of ['unbound', 'alice'] as const) expect({ who, get_page: (await call(who, 'get_page', { slug: page })).text.includes(canaryOf(k, 'digest')) }).toEqual({ who, get_page: false });}, 240_000);- test('a stamp that fails parks the origin: never purged, named by doctor with the page that blocks it, the derived page stays hidden; fixed, the next purge stamps and purges', async () => {+ test('a derived page that expires with its origin but is kept by another hold keeps the origin; restored, it is stamped before the origin goes', async () => {+ const k = 'cohold';+ await seed(k);+ const origin = slugOf(k, 'origin'), digest = slugOf(k, 'digest');+ await softDelete(origin);+ await softDelete(digest);+ await backdate(origin);+ await backdate(digest);+ const held = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true, only: page => page.slug !== digest });+ expect(held.slugs).not.toContain(origin);+ expect(held.slugs).not.toContain(digest);+ expect(held.blocked.find(b => b.slug === origin)?.code).toBe('derived_origin_parked');+ expect((await row(origin))?.deleted_at).not.toBeNull();+ const restored = await localCall('restore_page', { slug: digest, request_id: randomUUID(), expected_revision: await revision(digest) });+ expect(restored.isError ? restored.text : '').toBe('');+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);+ expect((await fm(digest)).visibility).toBe('private');+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);+ __resetPrivateVisibilityCacheForTests();+ for (const who of ['unbound', 'alice'] as const) expect({ who, seen: await seen(who, k, 'digest') }).toEqual({ who, seen: none });+ }, 240_000);++ test('an origin that expires within the next day is carried by this purge, so it cannot cross the cutoff uncarried mid-run', async () => {+ const k = 'lead';+ await seed(k);+ const origin = slugOf(k, 'origin');+ await softDelete(origin);+ await engine.executeRaw("UPDATE pages SET deleted_at = now() - interval '71 hours' WHERE source_id=$1 AND slug=$2", [SRC, origin]);+ const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });+ expect(result.slugs).not.toContain(origin);+ for (const shape of CARRIED) expect({ shape, visibility: (await fm(slugOf(k, shape))).visibility }).toEqual({ shape, visibility: 'private' });+ await backdate(origin);+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);+ }, 240_000);++ test('a consented manual purge whose only expired page is a fixed parked origin retries the label and purges it', async () => {+ const k = 'manual';+ await seed(k);+ const origin = slugOf(k, 'origin'), blocking = slugOf(k, 'digest');+ const checkout = mkdtempSync(join(tmpdir(), 'gbrain-origin-carry-checkout-'));+ await engine.executeRaw('UPDATE sources SET local_path=$2 WHERE id=$1', [SRC, checkout]);+ await engine.executeRaw('UPDATE pages SET source_path=$3 WHERE source_id=$1 AND slug=$2', [SRC, blocking, blocking + '.md']);+ await engine.setConfig('sync.write_through', 'false');+ try {+ await softDelete(origin);+ await backdate(origin);+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);+ } finally {+ await engine.setConfig('sync.write_through', 'true');+ await engine.executeRaw('UPDATE pages SET source_path=NULL WHERE source_id=$1 AND slug=$2', [SRC, blocking]);+ await engine.executeRaw('UPDATE sources SET local_path=NULL WHERE id=$1', [SRC]);+ rmSync(checkout, { recursive: true, force: true });+ }+ expect((await selectPurgeCandidates(engine, 72)).count).toBe(0);+ const result = await consentedPurgeDeletedPages(engine, 72, true);+ expect(result.slugs).toContain(origin);+ expect((await fm(blocking)).visibility).toBe('private');+ await nothingKept(origin);+ }, 240_000);++ test('a stamp that fails parks the origin: never purged, named by doctor with the page that blocks it, the derived page stays hidden; fixed, the next purge stamps and unparks and the one after purges', async () => {const k = 'park';await seed(k);const base = await baseline(k);@@ -331,10 +392,12 @@ for (const kind of testBackends()) {await engine.executeRaw('UPDATE sources SET local_path=NULL WHERE id=$1', [SRC]);rmSync(checkout, { recursive: true, force: true });}- const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });- expect(result.slugs).toContain(origin);- await nothingKept(origin);+ // Fixed: the next purge labels the page and unparks the origin but keeps it, since it was parked when that purge+ // began (so it was in no preview); the one after purges it.+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);expect((await checkDerivedOriginParked(engine)).status).toBe('ok');+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);+ await nothingKept(origin);await expectCarried(k, base.before, base.state, base.proposals);}, 240_000);});