Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Keep a private origin until every derived page it hides is labelled, including held tombstones and origins near the cutoff

garrytana77e769e610d
diff --git a/src/commands/doctor/checks/purge-parks.ts b/src/commands/doctor/checks/purge-parks.ts
index cbccd81806..2f95bce236 100644
--- a/src/commands/doctor/checks/purge-parks.ts
+++ b/src/commands/doctor/checks/purge-parks.ts
@@ -65,7 +65,7 @@ export async function checkDerivedOriginParked(engine: BrainEngine): Promise<Che
return {
name: ORIGIN_NAME, status: 'warn', details: { parked: parks.length, truncated: parks.length > origins.length, origins },
message: `${parks.length} soft-deleted page(s) are kept from the purge, because a page derived from each reads as private only through it and could not be labelled private: ${listed}`
- + `${parks.length > origins.length ? ` (first ${origins.length} of ${parks.length} shown)` : ''}. Fix the cause each reason names, or give the blocking page its own visibility label; the next purge labels the rest and purges the origin.`,
+ + `${parks.length > origins.length ? ` (first ${origins.length} of ${parks.length} shown)` : ''}. Fix the cause each reason names, or give the blocking page its own visibility label; the next purge labels the rest and frees the origin, and the purge after removes it (a consented gbrain pages purge-deleted does both).`,
...(first ? { fix: agentFix(['gbrain', 'repair', 'visibility', '--source', first.source_id, '--slug', first.blocking_slug!],
'Previews giving the blocking page its own visibility label; nothing is written.', ORIGIN_NAME) } : {}),
};
diff --git a/src/commands/pages.ts b/src/commands/pages.ts
index dbbbb17e19..dba0e03d4a 100644
--- a/src/commands/pages.ts
+++ b/src/commands/pages.ts
@@ -47,6 +47,7 @@ async function runPurgeDeleted(engine: BrainEngine, args: string[]): Promise<voi
const { olderThanHours, dryRun, json, retryParked } = parsePurgeArgs(args);
// #6741: one selector for the preview, the consent and the run (selectPurgeCandidates): the same cutoff, rename
// holds, writer state and parks; --retry-parked puts the parked pages the cutoff selects back among the candidates.
+ if (!dryRun) await (await import('../core/repair/origin-carry.ts')).retryParkedOrigins(engine, olderThanHours);
const selection = await selectPurgeCandidates(engine, olderThanHours, { retryParked });
const retried = selection.parked.filter(park => park.retried).length;
const notes = { ...(selection.parked.length ? { parked: selection.parked, retry_parked: retryParked } : {}), ...(selection.blocked.length ? { blocked: selection.blocked } : {}) };
diff --git a/src/core/persistence/purge-deleted.ts b/src/core/persistence/purge-deleted.ts
index 9c4b0e3e3d..39c4a8045a 100644
--- a/src/core/persistence/purge-deleted.ts
+++ b/src/core/persistence/purge-deleted.ts
@@ -14,7 +14,7 @@ import { linkStreamSlugs } from './page-purge.ts';
import { identityDowngradePurgeHold } from '../sync-identity-downgrades.ts';
import { atomRetirementPurgeHold } from './atom-retirement-hold.ts';
import { derivedRetirementPurgeHold } from './derived-retirement-hold.ts';
-import { DERIVED_ORIGIN_PARKED_CODE, derivedOriginPurgeHold, type DerivedOriginPark } from './derived-origin-hold.ts';
+import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, derivedOriginPurgeHold, type DerivedOriginPark } from './derived-origin-hold.ts';
import { writerOnboardingPreflight } from './onboarding.ts';
import { clearPurgePark, isParked, notePurgeSourceWriter, PURGE_PARK_LIMIT, purgeParkKey, prunePurgeParks, readPurgeParks, recordPurgeFailure, type PurgePark } from './purge-parks.ts';
import { ItemRun, type ItemFailure } from '../item-failure.ts';
@@ -90,6 +90,7 @@ export function purgeConsentRequest(count: number, olderThanHours: number, json:
/** The `purge_deleted_pages` op (`gbrain call`): the D6 consent first (`yes` is `--yes`); an empty purge set needs none. */
export async function consentedPurgeDeletedPages(engine: BrainEngine, olderThanHours: number, yes: boolean) {
+ await (await import('../repair/origin-carry.ts')).retryParkedOrigins(engine, olderThanHours);
const preview = await selectPurgeCandidates(engine, olderThanHours);
if (preview.count === 0) return { status: 'purged', count: 0, slugs: [] };
await requireConsent(purgeConsentRequest(preview.count, olderThanHours, false, yes ? ['--yes'] : [], false, 0, preview.sync_unreviewed));
@@ -331,8 +332,10 @@ export function purgePhaseNote(result: DeletedPagePurge): string {
export type PurgeFilter = (page: { source_id: string; slug: string }) => boolean;
export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter } = {}): Promise<DeletedPagePurge> {
- const origins = originParked(await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours));
+ const carried = await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours);
+ const origins = originParked(carried.parks);
const result = await purgeExpired(engine, olderThanHours, opts);
+ await clearDerivedOriginParks(engine, carried.unparked);
return origins.length ? { ...result, blocked: [...origins, ...result.blocked], origin_parked: origins.length } : result;
}
diff --git a/src/core/repair/origin-carry.ts b/src/core/repair/origin-carry.ts
index 171ffdc072..510c3d56b3 100644
--- a/src/core/repair/origin-carry.ts
+++ b/src/core/repair/origin-carry.ts
@@ -20,7 +20,7 @@ import { serializePageToMarkdown } from '../markdown.ts';
import { isSegmentedPage } from '../page-body/page-shape.ts';
import type { LabelEdit } from '../persistence/label-edit.ts';
import { maintenancePreflight, publishMaintenancePages } from '../persistence/prepared-maintenance.ts';
-import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, parkDerivedOrigin, pruneDerivedOriginParks, type DerivedOriginPark } from '../persistence/derived-origin-hold.ts';
+import { clearDerivedOriginParks, DERIVED_ORIGIN_PARKED_CODE, parkDerivedOrigin, pruneDerivedOriginParks, readDerivedOriginParks, type DerivedOriginPark } from '../persistence/derived-origin-hold.ts';
import { privacyPartSql, readHonorAliases, withoutCallerAccess } from '../search/private-visibility.ts';
import { filesInCheckout, importAtRevision, writeLabelFile } from './label-write.ts';
import { dependentsSql } from './private-labels.ts';
@@ -152,26 +152,30 @@ export async function deletePageCarryingPrivate(ctx: OperationContext, p: Record
if (origin && result.status === 'purged') await clearDerivedOriginParks(ctx.engine, [origin.id]);
return result;
}
+ await parkDerivedOrigin(ctx.engine, parked(origin, block));
const result = origin.deleted ? { status: 'held', slug: origin.slug, source_id: sourceId, request_id: p.request_id }
: await submitPageMutation(ctx, { operation: 'delete_page', params: { ...p, purge: false, request_id: derivedRequestId(String(p.request_id)) } });
- await parkDerivedOrigin(ctx.engine, parked(origin, block));
return { ...result, purge_held: { code: DERIVED_ORIGIN_PARKED_CODE,
message: `Not purged: ${block.slug ?? 'a page derived from it'} is private only through ${origin.slug} and could not be labelled private (${block.code ?? 'error'}: ${block.reason}). `
+ `${origin.slug} is soft-deleted and parked: no purge removes it until that page carries its own label; gbrain doctor names it as derived_origin_parked.` } };
}
+/** Hours before its cutoff a tombstone is carried, so none crosses the cutoff while the carry runs and is purged uncarried. */
+export const CARRY_LEAD_HOURS = 24;
+
/**
- * Before a purge: the expired soft-deleted origins get their dependents labelled private, judged against every expired
- * origin at once (a page private through two of them opens only when both go); each one still blocked is parked (the
- * purge skips it) and the rest lose any earlier park. The parks written this run.
+ * Before a purge: the soft-deleted origins expired or expiring within CARRY_LEAD_HOURS get their dependents labelled
+ * private, judged against all of them at once (a page private through two of them opens only when both go); each one
+ * still blocked is parked (the purge skips it). The parks written this run, and the origins whose earlier park no longer
+ * applies: the caller clears those after its purge, so an origin parked when the purge was previewed stays this run.
*/
-export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHours: number): Promise<DerivedOriginPark[]> {
+export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHours: number): Promise<{ parks: DerivedOriginPark[]; unparked: number[] }> {
await pruneDerivedOriginParks(engine);
const origins = await withoutCallerAccess(async () => {
await readHonorAliases(engine);
return engine.executeRaw<{ id: number; source_id: string; slug: string }>(`SELECT p.id, p.source_id, p.slug FROM pages p
WHERE p.deleted_at IS NOT NULL AND p.deleted_at < now() - ($1 || ' hours')::interval AND ${originSql('p')} ORDER BY p.id`,
- [String(Math.max(0, Math.floor(olderThanHours)))], bounded);
+ [String(Math.max(0, Math.floor(olderThanHours) - CARRY_LEAD_HOURS))], bounded);
});
const parks: DerivedOriginPark[] = [];
const batch = `derived-origin-carry:purge-${randomUUID()}`;
@@ -183,6 +187,14 @@ export async function carryPrivateBeforePurge(engine: BrainEngine, olderThanHour
await parkDerivedOrigin(engine, parked(origin, block));
parks.push({ ...parked(origin, block), parked_at: new Date().toISOString() });
}
- await clearDerivedOriginParks(engine, expired.filter(origin => !blocked.has(origin.id)).map(origin => origin.id));
- return parks;
+ return { parks, unparked: expired.filter(origin => !blocked.has(origin.id)).map(origin => origin.id) };
+}
+
+/**
+ * A consented manual purge first retries the label for every parked origin and unparks the ones it frees, so its preview
+ * and consent list them (otherwise a brain whose only expired pages are parked origins never gets past "nothing to purge").
+ */
+export async function retryParkedOrigins(engine: BrainEngine, olderThanHours: number): Promise<void> {
+ if (!(await readDerivedOriginParks(engine)).length) return;
+ await clearDerivedOriginParks(engine, (await carryPrivateBeforePurge(engine, olderThanHours)).unparked);
}
diff --git a/src/core/repair/private-labels.ts b/src/core/repair/private-labels.ts
index b4de44fa36..e7ff4c9f75 100644
--- a/src/core/repair/private-labels.ts
+++ b/src/core/repair/private-labels.ts
@@ -238,7 +238,8 @@ async function planRestore(engine: BrainEngine, scope: RepairScope, selection: S
* them go too), `keys` is unused, an atom also ties to the page a rename moved its `source_slug` to (slug_aliases and
* slug_rename_history, read before a purge scrubs them). A page that names readers is judged by the same privacy rules,
* since readable_by never relaxes them: it is returned when the removal would open it to its listed readers. A soft-deleted
- * page is a candidate too, since a restore brings it back with the origin row gone.
+ * page is a candidate too, since a restore brings it back with the origin row gone, and so is a removed page tied to
+ * another removed page (judged on its own row, against the postimage), since another hold may keep it from the purge.
*/
export function dependentsSql(mode: 'clear' | 'remove' = 'clear'): string {
const remove = mode === 'remove';
@@ -265,12 +266,13 @@ export function dependentsSql(mode: 'clear' | 'remove' = 'clear'): string {
UNION ALL SELECT r.id, h.cleared_id FROM hits h JOIN refs r ON r.source_id = h.source_id AND r.kind = 'depth' AND r.ref = h.slug
WHERE h.kind = 'declared'${live(' AND r.deleted_at IS NULL')}
UNION ALL SELECT p.id, t.cleared_id FROM provenance t JOIN pages p ON p.id = t.from_page_id AND p.source_id = t.source_id${live(' AND p.deleted_at IS NULL')}),
- candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)),
+ candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE ${remove ? 't.id <> t.cleared_id' : 'NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)'}),${remove ? `
+ candidate_rows AS MATERIALIZED (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM pages a WHERE a.id IN (SELECT id FROM candidates)),` : ''}
public_after AS MATERIALIZED (WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,
CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter
FROM pages pg LEFT JOIN cleared c ON c.id = pg.id${remove ? ' WHERE c.id IS NULL' : ''})
- SELECT after_clear.id FROM (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k
- JOIN pages a ON a.id = k.id OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})
+ SELECT after_clear.id FROM (${remove ? 'SELECT * FROM candidate_rows' : `SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k
+ JOIN pages a ON a.id = k.id`} OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})
SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'
WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category
FROM candidates cand JOIN pages p ON p.id = cand.id
diff --git a/test/derived-origin-carry-private.test.ts b/test/derived-origin-carry-private.test.ts
index 9faf57efb6..d3e6b48805 100644
--- a/test/derived-origin-carry-private.test.ts
+++ b/test/derived-origin-carry-private.test.ts
@@ -33,7 +33,7 @@ import { dispatchToolCall } from '../src/mcp/dispatch.ts';
import { __resetPrivateVisibilityCacheForTests } from '../src/core/search/private-visibility.ts';
import { setOwner } from '../src/core/access/owner.ts';
import { bindCredential } from '../src/core/access/members.ts';
-import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';
+import { consentedPurgeDeletedPages, purgeDeletedPagesCoordinated, selectPurgeCandidates } from '../src/core/persistence/purge-deleted.ts';
import { checkDerivedOriginParked } from '../src/commands/doctor/checks/purge-parks.ts';
import type { AuthInfo } from '../src/core/ops/contract.ts';
import { testBackends } from './helpers/test-backends.ts';
@@ -270,10 +270,11 @@ for (const kind of testBackends()) {
.toContainEqual(expect.objectContaining({ slug: origin, blocking_slug: digest }));
const restored = await localCall('restore_page', { slug: digest, request_id: randomUUID(), expected_revision: await revision(digest) });
expect(restored.isError ? restored.text : '').toBe('');
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);
+ expect((await fm(digest)).visibility).toBe('private');
const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });
expect(result.slugs).toContain(origin);
await nothingKept(origin);
- expect((await fm(digest)).visibility).toBe('private');
__resetPrivateVisibilityCacheForTests();
for (const who of ['unbound', 'alice'] as const) expect({ who, seen: await seen(who, k, 'digest') }).toEqual({ who, seen: none });
expect({ owner: await seen('owner', k, 'digest') }).toEqual({ owner: every });
@@ -295,7 +296,67 @@ for (const kind of testBackends()) {
for (const who of ['unbound', 'alice'] as const) expect({ who, get_page: (await call(who, 'get_page', { slug: page })).text.includes(canaryOf(k, 'digest')) }).toEqual({ who, get_page: false });
}, 240_000);
- test('a stamp that fails parks the origin: never purged, named by doctor with the page that blocks it, the derived page stays hidden; fixed, the next purge stamps and purges', async () => {
+ test('a derived page that expires with its origin but is kept by another hold keeps the origin; restored, it is stamped before the origin goes', async () => {
+ const k = 'cohold';
+ await seed(k);
+ const origin = slugOf(k, 'origin'), digest = slugOf(k, 'digest');
+ await softDelete(origin);
+ await softDelete(digest);
+ await backdate(origin);
+ await backdate(digest);
+ const held = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true, only: page => page.slug !== digest });
+ expect(held.slugs).not.toContain(origin);
+ expect(held.slugs).not.toContain(digest);
+ expect(held.blocked.find(b => b.slug === origin)?.code).toBe('derived_origin_parked');
+ expect((await row(origin))?.deleted_at).not.toBeNull();
+ const restored = await localCall('restore_page', { slug: digest, request_id: randomUUID(), expected_revision: await revision(digest) });
+ expect(restored.isError ? restored.text : '').toBe('');
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);
+ expect((await fm(digest)).visibility).toBe('private');
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);
+ __resetPrivateVisibilityCacheForTests();
+ for (const who of ['unbound', 'alice'] as const) expect({ who, seen: await seen(who, k, 'digest') }).toEqual({ who, seen: none });
+ }, 240_000);
+
+ test('an origin that expires within the next day is carried by this purge, so it cannot cross the cutoff uncarried mid-run', async () => {
+ const k = 'lead';
+ await seed(k);
+ const origin = slugOf(k, 'origin');
+ await softDelete(origin);
+ await engine.executeRaw("UPDATE pages SET deleted_at = now() - interval '71 hours' WHERE source_id=$1 AND slug=$2", [SRC, origin]);
+ const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });
+ expect(result.slugs).not.toContain(origin);
+ for (const shape of CARRIED) expect({ shape, visibility: (await fm(slugOf(k, shape))).visibility }).toEqual({ shape, visibility: 'private' });
+ await backdate(origin);
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);
+ }, 240_000);
+
+ test('a consented manual purge whose only expired page is a fixed parked origin retries the label and purges it', async () => {
+ const k = 'manual';
+ await seed(k);
+ const origin = slugOf(k, 'origin'), blocking = slugOf(k, 'digest');
+ const checkout = mkdtempSync(join(tmpdir(), 'gbrain-origin-carry-checkout-'));
+ await engine.executeRaw('UPDATE sources SET local_path=$2 WHERE id=$1', [SRC, checkout]);
+ await engine.executeRaw('UPDATE pages SET source_path=$3 WHERE source_id=$1 AND slug=$2', [SRC, blocking, blocking + '.md']);
+ await engine.setConfig('sync.write_through', 'false');
+ try {
+ await softDelete(origin);
+ await backdate(origin);
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);
+ } finally {
+ await engine.setConfig('sync.write_through', 'true');
+ await engine.executeRaw('UPDATE pages SET source_path=NULL WHERE source_id=$1 AND slug=$2', [SRC, blocking]);
+ await engine.executeRaw('UPDATE sources SET local_path=NULL WHERE id=$1', [SRC]);
+ rmSync(checkout, { recursive: true, force: true });
+ }
+ expect((await selectPurgeCandidates(engine, 72)).count).toBe(0);
+ const result = await consentedPurgeDeletedPages(engine, 72, true);
+ expect(result.slugs).toContain(origin);
+ expect((await fm(blocking)).visibility).toBe('private');
+ await nothingKept(origin);
+ }, 240_000);
+
+ test('a stamp that fails parks the origin: never purged, named by doctor with the page that blocks it, the derived page stays hidden; fixed, the next purge stamps and unparks and the one after purges', async () => {
const k = 'park';
await seed(k);
const base = await baseline(k);
@@ -331,10 +392,12 @@ for (const kind of testBackends()) {
await engine.executeRaw('UPDATE sources SET local_path=NULL WHERE id=$1', [SRC]);
rmSync(checkout, { recursive: true, force: true });
}
- const result = await purgeDeletedPagesCoordinated(engine, 72, { automatic: true });
- expect(result.slugs).toContain(origin);
- await nothingKept(origin);
+ // Fixed: the next purge labels the page and unparks the origin but keeps it, since it was parked when that purge
+ // began (so it was in no preview); the one after purges it.
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).not.toContain(origin);
expect((await checkDerivedOriginParked(engine)).status).toBe('ok');
+ expect((await purgeDeletedPagesCoordinated(engine, 72, { automatic: true })).slugs).toContain(origin);
+ await nothingKept(origin);
await expectCarried(k, base.before, base.state, base.proposals);
}, 240_000);
});