Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(purge): never compare or remove a root sync metafile in a tombstone purge (#7613)
garrytanb72cc352cfd0
diff --git a/src/core/persistence/page-prepare.ts b/src/core/persistence/page-prepare.tsindex 4cd0ffd2de..2907693a33 100644--- a/src/core/persistence/page-prepare.ts+++ b/src/core/persistence/page-prepare.ts@@ -6,7 +6,7 @@ import { boundedReads } from './bounded-reads.ts';import { isQuarantined, quarantineOutcome } from '../quarantine.ts';import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs';import { join, relative, sep } from 'node:path';-import { matchesAnyGlob } from '../sync.ts';+import { isSyncMetafile, matchesAnyGlob } from '../sync.ts';import { syncExcludePatterns } from './sync-discovery.ts';import type { BrainEngine } from '../engine.ts';import type { GBrainConfig } from '../config.ts';@@ -352,7 +352,7 @@ export async function prepareFileTarget(engine: BrainEngine, row: Pick<WriteRequ&& !(await gitDurabilityPolicy(binding, root)).durable) return undefined;if (options.excludeTombstone && snapshot?.page.deleted_at) {const rel = relative(root, path).split(sep).join('/');- if (matchesAnyGlob(rel, options.excludeTombstone.patterns)) { options.excludeTombstone.onExcluded(rel); return undefined; }+ if (unimportedBySync(rel, options.excludeTombstone.patterns)) { options.excludeTombstone.onExcluded(rel); return undefined; }}// A prior file over the single-string ceiling is read on demand in pieces (large-file.ts), never whole; so is the file// of a page stored in segments, or of a streamed target, at any size (its comparison reads it piecewise anyway).@@ -500,13 +500,14 @@ export async function pageDatabaseOnlyPublication(engine: SqlEngine, row: WriteR/*** #6368: a tombstone whose file is under sync.exclude does not own that file (sync never reads it again), so its purge is* database-only: the file is neither compared nor removed. `excludedPath` names that file; the apply re-checks it.+ * #7613: the same holds for a file at a root sync metafile path (`unimportedBySync`), so every tombstone purge runs the check.*/async function prepareDeleteFileTarget(engine: BrainEngine, row: WriteRequest, snapshot: PageSnapshot, purge: boolean,options: Parameters<typeof prepareFileTarget>[5]): Promise<{ file: PreparedMutation['file']; excludedPath: string | null; shared: SharedOrigin | null }> {let excludedPath: string | null = null, shared: SharedOrigin | null = null;- const exclude = purge && snapshot.page.deleted_at != null ? await syncExcludePatterns(engine) : [];- const file = await prepareFileTarget(engine, row, snapshot, null, undefined, { ...options,- ...(exclude.length ? { excludeTombstone: { patterns: exclude, onExcluded: (path: string) => { excludedPath = path; } } } : {}),+ const excludeTombstone = purge && snapshot.page.deleted_at != null+ ? { patterns: await syncExcludePatterns(engine), onExcluded: (path: string) => { excludedPath = path; } } : undefined;+ const file = await prepareFileTarget(engine, row, snapshot, null, undefined, { ...options, ...(excludeTombstone ? { excludeTombstone } : {}),sharedOrigin: { onShared: value => { shared = value; } } });return { file, excludedPath, shared };}@@ -550,8 +551,16 @@ function sharedTombstoneChanged(row: WriteRequest, what: string): OperationError`Page ${row.slug}'s file in source ${row.source_id} was recorded by another live page when this retention purge was prepared.`, pageFix(row.source_id, row.slug));}+/**+ * #7613: whether sync never imports the file at `rel` (source-relative, `/`-separated): it is under sync.exclude, or it is a+ * sync metafile at the source root, which is scaffolding in every `sync.metafile_pages` mode. A nested metafile can be a page.+ */+function unimportedBySync(rel: string, exclude: string[]): boolean {+ return (!rel.includes('/') && isSyncMetafile(rel)) || matchesAnyGlob(rel, exclude);+}+async function assertStillSyncExcluded(tx: BrainEngine, row: WriteRequest, excludedPath: string): Promise<void> {- if (matchesAnyGlob(excludedPath, await syncExcludePatterns(tx))) return;+ if (unimportedBySync(excludedPath, await syncExcludePatterns(tx))) return;throw pageRefusal('source_changed', 'sync.exclude changed while the purge was prepared.', row,`${excludedPath} was excluded from sync when the purge of ${row.slug} was prepared and is not any more, so nothing was purged. Run the purge again; it re-reads the exclusion.`, pageFix(row.source_id, row.slug));}diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 57a2b89093..d33ebf431b 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -425,6 +425,7 @@ test/purge-fact.test.tstest/purge-global-marker.test.tstest/purge-missing-cache-6911.test.tstest/purge-read-surfaces.test.ts+test/purge-root-metafile-7613.test.tstest/purge-sync-tombstone-hold-7130.test.tstest/read-meta-diag.test.tstest/recall-page-fill.test.tsdiff --git a/test/purge-root-metafile-7613.test.ts b/test/purge-root-metafile-7613.test.tsnew file mode 100644index 0000000000..69d76056e6--- /dev/null+++ b/test/purge-root-metafile-7613.test.ts@@ -0,0 +1,78 @@+import { expect, test } from 'bun:test';+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';+import type { BrainEngine } from '../src/core/engine.ts';+import { parseMarkdown, serializeMarkdown } from '../src/core/markdown.ts';+import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';++interface LegacyFile { slug: string; path: string; page: string; disk?: string }++const note = (title: string, body: string) => serializeMarkdown({}, body, '', { type: 'note', title, tags: [] });++/**+ * Pages imported before the brain was managed (and before their path became a sync metafile), soft-deleted past the+ * 72-hour retention. Each file holds `disk` when given, else the page's own markdown, so the purge's comparison matches.+ */+function seedExpiredTombstones(files: LegacyFile[]) {+ return async ({ engine, root }: { engine: BrainEngine; root: string }) => {+ for (const file of files) {+ mkdirSync(dirname(join(root, file.path)), { recursive: true });+ writeFileSync(join(root, file.path), file.disk ?? file.page);+ const parsed = parseMarkdown(file.page, file.slug);+ await engine.putPage(file.slug, { type: parsed.type, title: parsed.title, compiled_truth: parsed.compiled_truth,+ timeline: parsed.timeline, frontmatter: parsed.frontmatter, source_path: file.path }, { sourceId: 'default' });+ }+ await engine.executeRaw("UPDATE pages SET deleted_at = now() - interval '4 days' WHERE source_id = 'default'");+ };+}++async function remainingSlugs(engine: BrainEngine): Promise<string[]> {+ const rows = await engine.executeRaw<{ slug: string }>("SELECT slug FROM pages WHERE source_id = 'default' ORDER BY slug");+ return rows.map(row => row.slug);+}++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;++ test(`[${backend}] #7613: purging an expired tombstone recorded at a root sync metafile keeps that file`, async () => {+ const agents = note('Agent instructions', 'Run the checks before every commit.');+ const readme = note('Topic overview', 'An overview of notes/topic.');+ const resolverPage = note('Old resolver', 'A routing table that was replaced.');+ const resolverDisk = '# Resolver\n\nThe current routing table, edited by hand.\n';+ const files: LegacyFile[] = [+ { slug: 'agents', path: 'AGENTS.md', page: agents },+ { slug: 'readme', path: 'README.md', page: readme },+ { slug: 'resolver', path: 'RESOLVER.md', page: resolverPage, disk: resolverDisk },+ ];+ await managedBrain(async ({ engine, root }) => {+ const result = await purgeDeletedPagesCoordinated(engine, 72);+ expect(['AGENTS.md', 'README.md', 'RESOLVER.md'].filter(path => !existsSync(join(root, path)))).toEqual([]);+ expect(readFileSync(join(root, 'AGENTS.md'), 'utf8')).toBe(agents);+ expect(readFileSync(join(root, 'README.md'), 'utf8')).toBe(readme);+ expect(readFileSync(join(root, 'RESOLVER.md'), 'utf8')).toBe(resolverDisk);+ expect(result.blocked).toEqual([]);+ expect([...result.slugs].sort()).toEqual(['agents', 'readme', 'resolver']);+ expect(await remainingSlugs(engine)).toEqual([]);+ }, { databaseUrl, setup: seedExpiredTombstones(files) });+ }, 120_000);++ test(`[${backend}] #7613: a nested metafile keeps its drift check and an ordinary root file is still removed`, async () => {+ const topicPage = note('Topic readme', 'What notes/topic held when it was imported.');+ const topicDisk = '# Topic\n\nEdited after the page was deleted.\n';+ const plain = note('Scratch', 'A root page that is not scaffolding.');+ const files: LegacyFile[] = [+ { slug: 'notes/topic/readme', path: 'notes/topic/README.md', page: topicPage, disk: topicDisk },+ { slug: 'scratch', path: 'scratch.md', page: plain },+ ];+ await managedBrain(async ({ engine, root }) => {+ const result = await purgeDeletedPagesCoordinated(engine, 72);+ expect(result.slugs).toEqual(['scratch']);+ expect(result.blocked).toEqual([expect.objectContaining({ slug: 'notes/topic/readme', code: 'source_changed' })]);+ expect(await remainingSlugs(engine)).toEqual(['notes/topic/readme']);+ expect(readFileSync(join(root, 'notes/topic/README.md'), 'utf8')).toBe(topicDisk);+ expect(existsSync(join(root, 'scratch.md'))).toBe(false);+ }, { databaseUrl, setup: seedExpiredTombstones(files) });+ }, 120_000);+}