Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(persistence): a declared page equal to its committed file keeps it in HEAD (#7617)

garrytandab7656c3124
diff --git a/src/commands/sources-git-gap.ts b/src/commands/sources-git-gap.ts
index dd1aaaa966..f710d95e7c 100644
--- a/src/commands/sources-git-gap.ts
+++ b/src/commands/sources-git-gap.ts
@@ -41,8 +41,8 @@ export type WorktreeState = typeof WORKTREE_STATES[number];
/** One page as the census classified it, handed to a caller that plans per page (G8 conversion, persistence/git-conversion.ts). */
export interface GitGapPage { id: string; slug: string; revision: string | null; path: string | null; state: PageState; segmented: boolean;
declared?: DeclaredClass; worktree?: WorktreeState; headBlob?: string; databaseAlsoMoved?: boolean; unparseable?: UnparseableReason;
- /** A declared page: whether its row records a file (`source_path`). */
- recordsFile?: boolean }
+ /** A declared page: whether its row records a file (`source_path`), and the size of its committed blob. */
+ recordsFile?: boolean; headSize?: number }
export interface GitGapSample { slug: string; path: string; state: PageState; fields?: string[]; declared?: DeclaredClass; basis?: 'import_record' | 'no_import_record'; worktree?: WorktreeState }
@@ -204,7 +204,7 @@ async function censusBatch(tx: BrainEngine, batch: Key[], census: GitGapCensus,
census.states.declared_database_only++;
const oid = ctx.collect && blob && path ? await worktreeBlobOid(path, checkout!.head.format) : undefined;
ctx.collect?.(collected(key, rel, 'declared_database_only', { declared, recordsFile: Boolean(key.source_path),
- ...(blob ? { headBlob: blob.oid, worktree: oid === null ? 'deleted' : oid === blob.oid ? 'clean' : 'modified' } : {}) }));
+ ...(blob ? { headBlob: blob.oid, headSize: blob.size, worktree: oid === null ? 'deleted' : oid === blob.oid ? 'clean' : 'modified' } : {}) }));
if (blob) { census.declared_with_head_blob++; note(census, ctx.sample, { slug: key.slug, path: rel!, state: 'declared_database_only', declared }); }
else if (checkout) noFile.declared.push(key);
continue;
diff --git a/src/core/persistence/git-conversion.ts b/src/core/persistence/git-conversion.ts
index 79b6841a6a..c1a42641a0 100644
--- a/src/core/persistence/git-conversion.ts
+++ b/src/core/persistence/git-conversion.ts
@@ -10,8 +10,10 @@
* writes the page's export view (readExportPage, withdrawals overlaid: exactly what the census compares) to its file
* and commits it with commitThroughDoor, one commit per group of CONVERSION_GROUP_MAX pages, under the worktree writer
* lock, which is dropped between groups. No row changes: the database already holds those bytes.
- * - declared: database-only on purpose; nothing is written and nothing deleted.
- * - withhold: a private, alias-private or lineage-private page whose row still records a file. The apply takes G7a's
+ * - declared: database-only on purpose; nothing is written and nothing deleted. A declared page whose committed file
+ * reads back equal to it (the census contract) keeps that file in HEAD and is settled: a strict import changes nothing.
+ * - withhold: a private, alias-private or lineage-private page whose row still records a file that is absent from HEAD
+ * or differs from it. The apply takes G7a's
* private transition: a committed file's removal goes through the door, then the row withholds its origin
* (withheld_source_path), so the strict pass leaves the page live; with no file at HEAD, only the row changes. Any
* other declared page with a committed file is a discard (git_overwrites_database), as Git's import would replace it.
@@ -30,7 +32,7 @@
* The apply re-plans and refuses preview_changed on a different hash, adopts and withholds group by group (a crash leaves the source
* non-strict and the rerun preview lists only what is left), retries or reclassifies the parked effects, checks the
* strict level once more as it will read with the planned holds cleared (an overwrite nobody accepted stops it, and the
- * holds stay), clears the holds, runs the full strict-equivalent
+ * holds stay; a page settled as equal to its file that moved since is judged again), clears the holds, runs the full strict-equivalent
* pass (SyncOpts.canonicalOverride, internal) and sets `canonical: git` last, in the transaction that records the
* conversion. Writes can land between that pass and the flag, so it then runs one genuine strict pass and plans again:
* only a plan with no work left is a completed conversion; otherwise the source stays strict (Git wins) and the
@@ -47,7 +49,7 @@ import { currentSubmissionAuthority } from '../minions/submission-authority.ts';
import { inlineExportMarkdown, readExportPage, readExportPages, readExportWithdrawals, type ExportSnapshot } from '../export-snapshot.ts';
import { scannerSlugRootMode, scannerSourcePath } from '../write-through.ts';
import { censusGitGap, type GitGapCensus, type GitGapPage } from '../../commands/sources-git-gap.ts';
-import { compareInlineExportFile, ignoredPaths, type DeclaredClass, type ExportFileComparison } from './git-gap.ts';
+import { compareInlineExportFile, ignoredPaths, readBlobs, type DeclaredClass, type ExportFileComparison } from './git-gap.ts';
import { commitThroughDoor } from './git-door.ts';
import { currentVerifiedLocalWriter, localHostId } from './identity.ts';
import { acquireWorktree, readManagedMode } from './ownership.ts';
@@ -71,6 +73,7 @@ export const CONVERSION_OP = 'git-canonical-conversion';
const DOCS = 'docs/designs/GIT_CANONICAL_MEMORY.md#conversion';
const SAMPLE = 20;
const LOCK_WAIT_MS = 30_000;
+const MOVED_BATCH = 10_000;
export const ADOPT_STATES = ['database_ahead', 'undeclared_database_only', 'worktree_only'] as const;
export type AdoptState = typeof ADOPT_STATES[number];
@@ -116,7 +119,8 @@ export interface GitConversionPreview {
pages: number;
equal: number;
adopt: Counted<AdoptState> & { action: 'commit_through_door'; groups: number };
- declared: Counted<string> & { action: 'none' };
+ /** `committed_equal`: declared pages whose committed file reads back equal to the page, so the file stays in HEAD. */
+ declared: Counted<string> & { action: 'none'; committed_equal: number };
git_wins: Counted<GitWinsReason> & { action: 'index_committed_blob' };
/** G7a's private transition for a privacy-class page that still records a file: its file's removal is committed, then the row withholds its origin. */
withhold: Counted<WithholdKind> & { action: 'commit_removal_and_withhold'; groups: number };
@@ -132,9 +136,13 @@ export interface GitConversionPreview {
apply: string[];
}
-/** `settled`: paths whose Git import replaces nothing (the census reads them equal, or the plan adopts, withholds or accepts them). */
+interface PageRevision { id: string; revision: string | null }
+/**
+ * `settled`: paths whose Git import replaces nothing (the census reads them equal, or the plan adopts, withholds or accepts
+ * them); `compared`: the paths settled because the page read equal to its committed file, with the revision compared.
+ */
export interface GitConversionPlan { preview: GitConversionPreview; context: ManagedSyncContext; adopt: AdoptItem[]; withhold: WithholdItem[]; holds: HoldItem[]; parked: ParkedItem[];
- settled: Set<string>; incarnation: string }
+ settled: Set<string>; compared: Map<string, PageRevision>; incarnation: string }
const counted = <K extends string>(keys: readonly K[]): Counted<K> => ({ count: 0, by: Object.fromEntries(keys.map(key => [key, 0])) as Record<K, number>, sample: [] });
const bump = (into: Record<string, number>, key: string) => { into[key] = (into[key] ?? 0) + 1; };
@@ -165,10 +173,11 @@ const strictContext = (context: ManagedSyncContext): ManagedSyncContext => ({ ..
export async function planGitConversion(engine: BrainEngine, sourceId: string): Promise<GitConversionPlan> {
const context = await resolveManagedSyncContext(engine, { sourceId, noPull: true, dryRun: true });
const pages: GitGapPage[] = [];
- const equal = new Set<string>();
- const exported = new ExportChecks(context.root, await scannerSlugRootMode(engine, sourceId, context.root));
+ const equal = new Map<string, PageRevision>();
+ const mode = await scannerSlugRootMode(engine, sourceId, context.root);
+ const exported = new ExportChecks(context.root, mode);
const census = await censusGitGap(engine, sourceId, { sample: 0, collect: (page, snapshot) => {
- if (page.state === 'equal') { if (page.path) equal.add(page.path); return; }
+ if (page.state === 'equal') { if (page.path) equal.set(page.path, { id: page.id, revision: page.revision }); return; }
pages.push(page);
if (snapshot && page.state === 'database_ahead' && page.path && !page.segmented) exported.add(page.id, page.path, snapshot);
} });
@@ -184,6 +193,9 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
const adopt: AdoptItem[] = [], withhold: WithholdItem[] = [], accepted = new Set<string>(), discarded = new Set<string>();
const discard = (category: DiscardCategory, path: string) => { tally(preview.discard, category, path); discarded.add(path); };
const gitWins = (reason: GitWinsReason, path: string) => { tally(preview.git_wins, reason, path); accepted.add(path); };
+ const declaredEqual = await equalDeclaredPaths(engine, sourceId, root, mode,
+ pages.filter(page => page.state === 'declared_database_only' && page.path && page.headBlob && !page.segmented));
+ preview.declared.committed_equal = declaredEqual.size;
const holds = holdRecords.map(hold => holdItem(hold));
for (const hold of holds) {
tally(preview.holds, hold.side ? `${hold.code}:${hold.side}` : hold.code, hold.path);
@@ -202,7 +214,9 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
switch (page.state) {
case 'declared_database_only': {
tally(preview.declared, page.declared!, path);
- // A privacy class takes G7a's private transition; any other declared page with a committed file stays a discard (planIndex).
+ // A committed file equal to the page stays in HEAD (a strict import of it changes nothing). Otherwise a privacy class
+ // takes G7a's private transition, and any other declared page with a committed file stays a discard (planIndex).
+ if (page.path && declaredEqual.has(page.path)) continue;
if (!isPrivacyClass(page.declared) || !page.path || (!page.headBlob && !page.recordsFile)) continue;
if (page.worktree === 'modified') { discard('uncommitted_edit', path); bump(preview.discard.detail.uncommitted_edit, 'modified'); continue; }
const disk = page.headBlob ? diskBytes(join(root, page.path)) : null;
@@ -241,7 +255,7 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
preview.parked_effects.count = parked.length;
preview.parked_effects.retry = parked.filter(item => item.action === 'retry').length;
preview.parked_effects.reclassify = parked.filter(item => item.action === 'reclassify').length;
- const settled = new Set([...equal, ...adopt.map(item => item.path), ...withhold.map(item => item.path), ...accepted]);
+ const settled = new Set([...equal.keys(), ...declaredEqual, ...adopt.map(item => item.path), ...withhold.map(item => item.path), ...accepted]);
const placed = new Set([...settled, ...discarded]);
const unplaced = holds.filter(hold => !placed.has(hold.path));
for (const hold of unplaced) bump(preview.holds.unplaced.by, hold.side ? `${hold.code}:${hold.side}` : hold.code);
@@ -257,14 +271,16 @@ export async function planGitConversion(engine: BrainEngine, sourceId: string):
discard: preview.discard.by.ignore_status_unknown ? preview.discard.by : { ...preview.discard.by, ignore_status_unknown: undefined },
// Bound only when present, so the hash of a plan with nothing to withhold is the one earlier releases gave.
withhold: withhold.length ? withhold.map(item => [item.id, item.revision, item.path, item.kind, item.disk]) : undefined,
+ declared_equal: declaredEqual.size ? pages.filter(page => page.path && declaredEqual.has(page.path)).map(page => [page.id, page.revision, page.path]) : undefined,
holds: holds.map(hold => [hold.path, hold.code, hold.side, hold.observed_at]), parked: parked.map(item => [item.effect, item.action, item.codes]), index: preview.index });
preview.apply = ['gbrain', 'sources', 'canonical', sourceId, '--to', 'git', '--apply', '--expect', preview.preview_hash];
- return { preview, context, adopt, withhold, holds, parked, settled, incarnation };
+ const compared = new Map([...equal, ...pages.flatMap(page => page.path && declaredEqual.has(page.path) ? [[page.path, { id: page.id, revision: page.revision }] as const] : [])]);
+ return { preview, context, adopt, withhold, holds, parked, settled, compared, incarnation };
}
function emptyPreview(sourceId: string, context: ManagedSyncContext, census: GitGapCensus): GitConversionPreview {
return { status: 'preview', source_id: sourceId, to: 'git', strict: context.source.config?.canonical === 'git', head: census.head, pages: census.pages, equal: census.states.equal,
- adopt: { ...counted(ADOPT_STATES), action: 'commit_through_door', groups: 0 }, declared: { ...counted<string>([]), action: 'none' },
+ adopt: { ...counted(ADOPT_STATES), action: 'commit_through_door', groups: 0 }, declared: { ...counted<string>([]), action: 'none', committed_equal: 0 },
git_wins: { ...counted(GIT_WINS_REASONS), action: 'index_committed_blob' }, withhold: { ...counted(WITHHOLD_KINDS), action: 'commit_removal_and_withhold', groups: 0 }, holds: { ...counted<string>([]), action: 'clear', unplaced: { count: 0, by: {} } },
parked_effects: { count: 0, retry: 0, reclassify: 0, by_code: {} }, index: { imports: 0, deletes: 0 }, discard: emptyDiscard(), remaining: 0, preview_hash: '', apply: [] };
}
@@ -337,6 +353,33 @@ async function planAdoption(engine: BrainEngine, sourceId: string, root: string,
}
}
+/**
+ * The declared pages whose committed file reads back equal to the page's export view under the census contract (#7618): a
+ * strict import of that file changes nothing, so the conversion leaves the file in HEAD and the page settled. Read
+ * ADOPTION_BATCH at a time in read-only transactions.
+ */
+async function equalDeclaredPaths(engine: BrainEngine, sourceId: string, root: string, mode: Awaited<ReturnType<typeof scannerSlugRootMode>>,
+ pages: GitGapPage[]): Promise<Set<string>> {
+ const equal = new Set<string>();
+ if (!pages.length) return equal;
+ const withdrawals = await readExportWithdrawals(engine, sourceId);
+ for (let start = 0; start < pages.length; start += ADOPTION_BATCH) {
+ const batch = pages.slice(start, start + ADOPTION_BATCH);
+ const blobs = await readBlobs(root, batch.map(page => ({ oid: page.headBlob!, size: page.headSize ?? 0, mode: '100644' })));
+ if (!blobs) throw opError('storage_error', `Git could not read committed files of source ${sourceId}'s checkout, so nothing was planned.`,
+ `Check the checkout with git -C ${root} fsck, then preview again.`);
+ const snapshots = await engine.transaction(async tx => {
+ await tx.executeRaw('SET TRANSACTION READ ONLY');
+ return readExportPages(tx, batch.map(page => ({ ...page, source_id: sourceId })), withdrawals);
+ });
+ batch.forEach((page, i) => {
+ const comparison = compareInlineExportFile(blobs.get(page.headBlob!)!, scannerSourcePath(root, join(root, page.path!), mode), snapshots[i]!);
+ if (comparison.state === 'equal') equal.add(page.path!);
+ });
+ }
+ return equal;
+}
+
/** A level path (Git-root relative) as the census names it (source-root relative). */
const sourcePath = (scope: string, gitPath: string) => scope && gitPath.startsWith(`${scope}/`) ? gitPath.slice(scope.length + 1) : gitPath;
@@ -415,13 +458,15 @@ export async function applyGitConversion(engine: BrainEngine, sourceId: string,
const withheld = await withholdGroups(engine, plan, previewCommand);
const parked = await settleParked(engine, sourceId, plan.parked);
// The planned holds are cleared only once this check passes, so a refusal keeps them: a held path is judged as the
- // strict pass will see it once its hold is cleared.
+ // strict pass will see it once its hold is cleared. A page the plan settled as equal to its file is judged again once it
+ // moved since (its equality no longer holds).
const held = new Set(plan.holds.map(hold => hold.path));
+ const moved = await movedSincePlan(engine, sourceId, plan.compared);
const recheck = await readShadowLevel(engine, { sourceId, full: true, context: strictContext(plan.context) });
const preview = { ...plan.preview, discard: emptyDiscard() };
for (const entry of recheck?.paths ?? []) {
const path = sourcePath(recheck!.scope, entry.path);
- if (plan.settled.has(path)) continue;
+ if (plan.settled.has(path) && !moved.has(path)) continue;
const result = held.has(path) ? classifySyncPath({ ...entry.state, hold: null }, { strict: true }) : entry.result;
if (!result.settled && !result.inFlight && result.gitWins) tally(preview.discard, 'git_overwrites_database', path);
}
@@ -440,6 +485,20 @@ export async function applyGitConversion(engine: BrainEngine, sourceId: string,
return { status: after.remaining === 0 ? 'converted' : 'incomplete', source_id: sourceId, head: after.head, committed, withheld, holds_cleared: holdsCleared, parked, after };
}
+/** The paths of `compared` whose page has another revision than the plan compared, or is gone. */
+async function movedSincePlan(engine: BrainEngine, sourceId: string, compared: Map<string, PageRevision>): Promise<Set<string>> {
+ const moved = new Set<string>();
+ const entries = [...compared];
+ for (let start = 0; start < entries.length; start += MOVED_BATCH) {
+ const batch = entries.slice(start, start + MOVED_BATCH);
+ const rows = await engine.executeRaw<{ id: string; revision: string | null }>(`SELECT id::text AS id, knowledge_revision::text AS revision FROM pages
+ WHERE source_id=$1 AND deleted_at IS NULL AND id = ANY(string_to_array($2::text, ',')::bigint[])`, [sourceId, batch.map(([, page]) => page.id).join(',')]);
+ const live = new Map(rows.map(row => [row.id, row.revision]));
+ for (const [path, page] of batch) if (!live.has(page.id) || live.get(page.id) !== page.revision) moved.add(path);
+ }
+ return moved;
+}
+
const PASS_DONE = new Set(['first_sync', 'synced', 'up_to_date']);
function passOrThrow(sourceId: string, result: { status: string; reason?: string }): void {
if (PASS_DONE.has(result.status)) return;
diff --git a/test/git-canonical-conversion-g8a.test.ts b/test/git-canonical-conversion-g8a.test.ts
index 39701ddaec..41ac959901 100644
--- a/test/git-canonical-conversion-g8a.test.ts
+++ b/test/git-canonical-conversion-g8a.test.ts
@@ -417,9 +417,10 @@ for (const backend of testBackends()) describe(`[${backend}] G8a: sources canoni
expect(again.preview_hash).toBe(preview.preview_hash);
}, { databaseUrl, setup }), 180_000);
- test('#7618: a private page with a committed file withholds it: the removal is committed and the page stays live and unchanged', () => managedBrain(async brain => {
+ test('#7618: a private page that differs from its committed file withholds it: the removal is committed and the page stays live and unchanged', () => managedBrain(async brain => {
const { engine, root } = brain;
await committedPrivatePage(brain);
+ await databaseOnlyEdit(engine, 'notes/private', 'A private note, edited in the database.');
const before = await pageRow(engine, 'notes/private'), base = git(root, 'rev-parse', 'HEAD');
const { preview } = await planGitConversion(engine, 'default');
expect(preview).toMatchObject({ declared: { by: { private: 1 } }, withhold: { count: 1, groups: 1, by: { file_committed: 1, file_absent: 0 }, sample: ['notes/private.md'] },
@@ -437,6 +438,23 @@ for (const backend of testBackends()) describe(`[${backend}] G8a: sources canoni
expect((await pageRow(engine, 'notes/private')).deleted).toBe(false);
}, { databaseUrl, setup }), 180_000);
+ test('#7618: a private page equal to its committed file is settled: no withhold, no discard, and the file stays in HEAD', () => managedBrain(async brain => {
+ const { engine, root } = brain;
+ await committedPrivatePage(brain);
+ // No import record, as a page imported before records were written has none: the strict level reads it unverified.
+ await dropImportRecord(engine, 'notes/private');
+ const before = await pageRow(engine, 'notes/private'), revision = (await page(engine, 'notes/private'))!.revision, base = git(root, 'rev-parse', 'HEAD');
+ const { preview } = await planGitConversion(engine, 'default');
+ expect(preview).toMatchObject({ declared: { by: { private: 1 }, committed_equal: 1 }, withhold: { count: 0 }, discard: { count: 0 }, remaining: 0 });
+ const result = await applyGitConversion(engine, 'default', preview.preview_hash, PROCESSING);
+ expect(result).toMatchObject({ status: 'converted', withheld: { pages: 0 }, after: { remaining: 0 } });
+ expect(await strict(engine)).toBe(true);
+ expect(git(root, 'rev-parse', 'HEAD')).toBe(base);
+ expect(atHead(root, 'notes/private.md')).toBe('notes/private.md');
+ expect(await pageRow(engine, 'notes/private')).toEqual(before);
+ expect((await page(engine, 'notes/private'))!.revision).toBe(revision);
+ }, { databaseUrl, setup }), 180_000);
+
test('#7618 regression: converting never deletes a private page whose file was removed by hand', () => managedBrain(async brain => {
const { engine, root } = brain;
// Removed and committed with no sync since: the strict pass would read a live page whose file a commit removed.
@@ -469,7 +487,7 @@ for (const backend of testBackends()) describe(`[${backend}] G8a: sources canoni
const { engine, root } = brain;
expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
await databaseOnlyEdit(engine, 'notes/ahead', 'Written to the database alone.');
- // notes/same: declared database-only, its row equal to its committed file, and held. The plan places nothing there.
+ // notes/same: declared database-only, its row equal to its committed file, and held. The plan settles it as equal.
await engine.transaction(tx => withCoordinatedWrite(tx, ['default'], () => tx.executeRaw(
"UPDATE pages SET database_only_reason='db_only' WHERE source_id='default' AND slug='notes/same'"), TEST_WRITE_ATTRIBUTION));
const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation::text AS incarnation FROM sources WHERE id='default'");
@@ -478,8 +496,8 @@ for (const backend of testBackends()) describe(`[${backend}] G8a: sources canoni
run_id: 'run-1', mode: 'managed', meta: { recovery_version: 1 } }));
const base = git(root, 'rev-parse', 'HEAD');
const { preview } = await planGitConversion(engine, 'default');
- expect(preview).toMatchObject({ adopt: { count: 1 }, holds: { count: 1, unplaced: { count: 1 } }, discard: { count: 0 } });
- // After the first group commits, a database edit lands on the held page: Git's import would now overwrite it.
+ expect(preview).toMatchObject({ adopt: { count: 1 }, declared: { committed_equal: 1 }, holds: { count: 1, unplaced: { count: 0 } }, discard: { count: 0 } });
+ // After the first group commits, a database edit lands on the held page: it no longer equals its file, and Git's import would overwrite it.
installFaultHook(async point => { if (point === 'conversion:group_committed') await databaseOnlyEdit(engine, 'notes/same', 'A late database edit.'); });
try {
await expect(applyGitConversion(engine, 'default', preview.preview_hash, PROCESSING)).rejects.toMatchObject({ code: 'git_conversion_blocked', detail: 'git_overwrites_database' });