Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(purge): never compare or remove a root sync metafile in a tombstone purge (#7613)

garrytanb72cc352cfd0
diff --git a/src/core/persistence/page-prepare.ts b/src/core/persistence/page-prepare.ts
index 4cd0ffd2de..2907693a33 100644
--- a/src/core/persistence/page-prepare.ts
+++ b/src/core/persistence/page-prepare.ts
@@ -6,7 +6,7 @@ import { boundedReads } from './bounded-reads.ts';
import { isQuarantined, quarantineOutcome } from '../quarantine.ts';
import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs';
import { join, relative, sep } from 'node:path';
-import { matchesAnyGlob } from '../sync.ts';
+import { isSyncMetafile, matchesAnyGlob } from '../sync.ts';
import { syncExcludePatterns } from './sync-discovery.ts';
import type { BrainEngine } from '../engine.ts';
import type { GBrainConfig } from '../config.ts';
@@ -352,7 +352,7 @@ export async function prepareFileTarget(engine: BrainEngine, row: Pick<WriteRequ
&& !(await gitDurabilityPolicy(binding, root)).durable) return undefined;
if (options.excludeTombstone && snapshot?.page.deleted_at) {
const rel = relative(root, path).split(sep).join('/');
- if (matchesAnyGlob(rel, options.excludeTombstone.patterns)) { options.excludeTombstone.onExcluded(rel); return undefined; }
+ if (unimportedBySync(rel, options.excludeTombstone.patterns)) { options.excludeTombstone.onExcluded(rel); return undefined; }
}
// A prior file over the single-string ceiling is read on demand in pieces (large-file.ts), never whole; so is the file
// of a page stored in segments, or of a streamed target, at any size (its comparison reads it piecewise anyway).
@@ -500,13 +500,14 @@ export async function pageDatabaseOnlyPublication(engine: SqlEngine, row: WriteR
/**
* #6368: a tombstone whose file is under sync.exclude does not own that file (sync never reads it again), so its purge is
* database-only: the file is neither compared nor removed. `excludedPath` names that file; the apply re-checks it.
+ * #7613: the same holds for a file at a root sync metafile path (`unimportedBySync`), so every tombstone purge runs the check.
*/
async function prepareDeleteFileTarget(engine: BrainEngine, row: WriteRequest, snapshot: PageSnapshot, purge: boolean,
options: Parameters<typeof prepareFileTarget>[5]): Promise<{ file: PreparedMutation['file']; excludedPath: string | null; shared: SharedOrigin | null }> {
let excludedPath: string | null = null, shared: SharedOrigin | null = null;
- const exclude = purge && snapshot.page.deleted_at != null ? await syncExcludePatterns(engine) : [];
- const file = await prepareFileTarget(engine, row, snapshot, null, undefined, { ...options,
- ...(exclude.length ? { excludeTombstone: { patterns: exclude, onExcluded: (path: string) => { excludedPath = path; } } } : {}),
+ const excludeTombstone = purge && snapshot.page.deleted_at != null
+ ? { patterns: await syncExcludePatterns(engine), onExcluded: (path: string) => { excludedPath = path; } } : undefined;
+ const file = await prepareFileTarget(engine, row, snapshot, null, undefined, { ...options, ...(excludeTombstone ? { excludeTombstone } : {}),
sharedOrigin: { onShared: value => { shared = value; } } });
return { file, excludedPath, shared };
}
@@ -550,8 +551,16 @@ function sharedTombstoneChanged(row: WriteRequest, what: string): OperationError
`Page ${row.slug}'s file in source ${row.source_id} was recorded by another live page when this retention purge was prepared.`, pageFix(row.source_id, row.slug));
}
+/**
+ * #7613: whether sync never imports the file at `rel` (source-relative, `/`-separated): it is under sync.exclude, or it is a
+ * sync metafile at the source root, which is scaffolding in every `sync.metafile_pages` mode. A nested metafile can be a page.
+ */
+function unimportedBySync(rel: string, exclude: string[]): boolean {
+ return (!rel.includes('/') && isSyncMetafile(rel)) || matchesAnyGlob(rel, exclude);
+}
+
async function assertStillSyncExcluded(tx: BrainEngine, row: WriteRequest, excludedPath: string): Promise<void> {
- if (matchesAnyGlob(excludedPath, await syncExcludePatterns(tx))) return;
+ if (unimportedBySync(excludedPath, await syncExcludePatterns(tx))) return;
throw pageRefusal('source_changed', 'sync.exclude changed while the purge was prepared.', row,
`${excludedPath} was excluded from sync when the purge of ${row.slug} was prepared and is not any more, so nothing was purged. Run the purge again; it re-reads the exclusion.`, pageFix(row.source_id, row.slug));
}
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 57a2b89093..d33ebf431b 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -425,6 +425,7 @@ test/purge-fact.test.ts
test/purge-global-marker.test.ts
test/purge-missing-cache-6911.test.ts
test/purge-read-surfaces.test.ts
+test/purge-root-metafile-7613.test.ts
test/purge-sync-tombstone-hold-7130.test.ts
test/read-meta-diag.test.ts
test/recall-page-fill.test.ts
diff --git a/test/purge-root-metafile-7613.test.ts b/test/purge-root-metafile-7613.test.ts
new file mode 100644
index 0000000000..69d76056e6
--- /dev/null
+++ b/test/purge-root-metafile-7613.test.ts
@@ -0,0 +1,78 @@
+import { expect, test } from 'bun:test';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { parseMarkdown, serializeMarkdown } from '../src/core/markdown.ts';
+import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+
+interface LegacyFile { slug: string; path: string; page: string; disk?: string }
+
+const note = (title: string, body: string) => serializeMarkdown({}, body, '', { type: 'note', title, tags: [] });
+
+/**
+ * Pages imported before the brain was managed (and before their path became a sync metafile), soft-deleted past the
+ * 72-hour retention. Each file holds `disk` when given, else the page's own markdown, so the purge's comparison matches.
+ */
+function seedExpiredTombstones(files: LegacyFile[]) {
+ return async ({ engine, root }: { engine: BrainEngine; root: string }) => {
+ for (const file of files) {
+ mkdirSync(dirname(join(root, file.path)), { recursive: true });
+ writeFileSync(join(root, file.path), file.disk ?? file.page);
+ const parsed = parseMarkdown(file.page, file.slug);
+ await engine.putPage(file.slug, { type: parsed.type, title: parsed.title, compiled_truth: parsed.compiled_truth,
+ timeline: parsed.timeline, frontmatter: parsed.frontmatter, source_path: file.path }, { sourceId: 'default' });
+ }
+ await engine.executeRaw("UPDATE pages SET deleted_at = now() - interval '4 days' WHERE source_id = 'default'");
+ };
+}
+
+async function remainingSlugs(engine: BrainEngine): Promise<string[]> {
+ const rows = await engine.executeRaw<{ slug: string }>("SELECT slug FROM pages WHERE source_id = 'default' ORDER BY slug");
+ return rows.map(row => row.slug);
+}
+
+for (const backend of testBackends()) {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+
+ test(`[${backend}] #7613: purging an expired tombstone recorded at a root sync metafile keeps that file`, async () => {
+ const agents = note('Agent instructions', 'Run the checks before every commit.');
+ const readme = note('Topic overview', 'An overview of notes/topic.');
+ const resolverPage = note('Old resolver', 'A routing table that was replaced.');
+ const resolverDisk = '# Resolver\n\nThe current routing table, edited by hand.\n';
+ const files: LegacyFile[] = [
+ { slug: 'agents', path: 'AGENTS.md', page: agents },
+ { slug: 'readme', path: 'README.md', page: readme },
+ { slug: 'resolver', path: 'RESOLVER.md', page: resolverPage, disk: resolverDisk },
+ ];
+ await managedBrain(async ({ engine, root }) => {
+ const result = await purgeDeletedPagesCoordinated(engine, 72);
+ expect(['AGENTS.md', 'README.md', 'RESOLVER.md'].filter(path => !existsSync(join(root, path)))).toEqual([]);
+ expect(readFileSync(join(root, 'AGENTS.md'), 'utf8')).toBe(agents);
+ expect(readFileSync(join(root, 'README.md'), 'utf8')).toBe(readme);
+ expect(readFileSync(join(root, 'RESOLVER.md'), 'utf8')).toBe(resolverDisk);
+ expect(result.blocked).toEqual([]);
+ expect([...result.slugs].sort()).toEqual(['agents', 'readme', 'resolver']);
+ expect(await remainingSlugs(engine)).toEqual([]);
+ }, { databaseUrl, setup: seedExpiredTombstones(files) });
+ }, 120_000);
+
+ test(`[${backend}] #7613: a nested metafile keeps its drift check and an ordinary root file is still removed`, async () => {
+ const topicPage = note('Topic readme', 'What notes/topic held when it was imported.');
+ const topicDisk = '# Topic\n\nEdited after the page was deleted.\n';
+ const plain = note('Scratch', 'A root page that is not scaffolding.');
+ const files: LegacyFile[] = [
+ { slug: 'notes/topic/readme', path: 'notes/topic/README.md', page: topicPage, disk: topicDisk },
+ { slug: 'scratch', path: 'scratch.md', page: plain },
+ ];
+ await managedBrain(async ({ engine, root }) => {
+ const result = await purgeDeletedPagesCoordinated(engine, 72);
+ expect(result.slugs).toEqual(['scratch']);
+ expect(result.blocked).toEqual([expect.objectContaining({ slug: 'notes/topic/readme', code: 'source_changed' })]);
+ expect(await remainingSlugs(engine)).toEqual(['notes/topic/readme']);
+ expect(readFileSync(join(root, 'notes/topic/README.md'), 'utf8')).toBe(topicDisk);
+ expect(existsSync(join(root, 'scratch.md'))).toBe(false);
+ }, { databaseUrl, setup: seedExpiredTombstones(files) });
+ }, 120_000);
+}