Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: rows-behind primes the metafile heads per batch so the eligibility screen does no Git lookup per page

garrytanb517c4c7d640
diff --git a/src/core/persistence/rows-behind.ts b/src/core/persistence/rows-behind.ts
index 439de1b4cc..f586aaa5df 100644
--- a/src/core/persistence/rows-behind.ts
+++ b/src/core/persistence/rows-behind.ts
@@ -118,7 +118,7 @@ export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
await readHonorAliases(engine);
const scope = relative(checkpoint.gitRoot, realpathSync(root)).split(sep).join('/');
const gitPath = (rel: string) => scope ? `${scope}/${rel}` : rel;
- const { eligible } = await loadSyncEligibility(engine, { root, gitRoot: checkpoint.gitRoot, sourceId, source: { last_commit: source.last_commit, config: source.config ?? {} } });
+ const { eligible, metafileHeads } = await loadSyncEligibility(engine, { root, gitRoot: checkpoint.gitRoot, sourceId, source: { last_commit: source.last_commit, config: source.config ?? {} } });
const policy = { mirror: false, databaseOnlySource: false, dbOnly: (slug: string) => isSourceDbOnlySlug(root, slug, 'refuse'), ignored: new Set<string>() };
const scan: RowsBehindScan = { ...empty(sourceId, 'known'), checkpoint: checkpoint.commit, head: head.commit };
const count = (why: RowsBehindSkip) => { scan.counts[why] = (scan.counts[why] ?? 0) + 1; };
@@ -142,21 +142,26 @@ export async function scanRowsBehind(engine: BrainEngine, sourceId: string,
if (!batch.length) return true;
cursor = batch[batch.length - 1]!.id;
ids = batch.map(key => Number(key.id));
- const rows: Array<{ key: Key; rel: string; blob: TreeBlob }> = [];
+ const listed: Array<{ key: Key; rel: string; blob: TreeBlob }> = [];
for (const key of batch) {
const path = canonicalPath(context, key);
if (!path) { skip(key.id, 'unsafe_path'); continue; }
const rel = relative(root, path).split(sep).join('/');
if (rel.startsWith('../') || rel === '..') { skip(key.id, 'unsafe_path'); continue; }
if (!isMarkdown(rel)) { skip(key.id, 'not_markdown'); continue; }
- if (!eligible(gitPath(rel))) { skip(key.id, 'not_eligible'); continue; }
const frontmatter = typeof key.frontmatter === 'string' ? JSON.parse(key.frontmatter) : key.frontmatter;
if (declaredDatabaseOnly({ ...key, frontmatter }, rel, policy)) { skip(key.id, 'declared_database_only'); continue; }
const blob = checkpoint.blobs.get(gitPath(rel));
if (!blob) { skip(key.id, 'no_file_at_checkpoint'); continue; }
if (blob.mode !== '100644' && blob.mode !== '100755') { skip(key.id, 'not_regular_file'); continue; }
- rows.push({ key, rel, blob });
+ listed.push({ key, rel, blob });
}
+ metafileHeads.prime(listed.map(row => ({ path: gitPath(row.rel), oid: row.blob.oid, size: row.blob.size })));
+ const rows = listed.filter(row => {
+ if (eligible(gitPath(row.rel))) return true;
+ skip(row.key.id, 'not_eligible');
+ return false;
+ });
if (!rows.length) return batch.length < ROWS_BEHIND_BATCH;
const held = await heldGitPaths(tx, sourceId, incarnation, rows.map(row => row.rel));
const records = await readSyncImportProvenances(tx, sourceId, incarnation, rows.map(row => Number(row.key.id)));
diff --git a/test/rows-behind-checkpoint-8071.test.ts b/test/rows-behind-checkpoint-8071.test.ts
index 48f9fa2c0e..0dceca697f 100644
--- a/test/rows-behind-checkpoint-8071.test.ts
+++ b/test/rows-behind-checkpoint-8071.test.ts
@@ -19,6 +19,7 @@
* - the scan checks its deadline inside a batch and reports how far it got; doctor counts unproven rows without the
* importer's confirmation; a hold row applies the refusal the apply computes;
* - a path sync does not read (`sync.exclude`) is not behind; an unreadable checkpoint commit is unknown, never clean;
+ * nested typed index pages are screened from the batch's checkpoint blobs, without a Git lookup per page;
* - the apply refuses a reimport row whose page moved since the preview and a hold row whose import is no longer refused;
* - repo-less: a brain with no checkout reports not_applicable (ok) and the preview lists nothing.
*
@@ -27,7 +28,8 @@
import { describe, expect, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
-import { mkdirSync, writeFileSync } from 'node:fs';
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
@@ -299,6 +301,19 @@ for (const backend of testBackends()) describe(`[${backend}] #8071: rows behind
expect(await doctor(engine)).toMatchObject({ name: CHECK, status: 'ok', readiness_state: 'unknown' });
}, { databaseUrl, setup }), 240_000);
+ test('nested typed index pages are screened with one batched head read per batch, not a Git read per page', () => managedBrain(async ({ engine, root }) => {
+ for (const dir of ['notes/s1', 'notes/s2', 'notes/s3']) { mkdirSync(join(root, dir), { recursive: true }); writeFileSync(join(root, dir, 'index.md'), note(dir, `The index of ${dir}.`)); }
+ commit(root, 'indexes');
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ const trace = join(tmpdir(), `gbrain-rows-behind-trace-${randomUUID()}.log`);
+ try {
+ const scan = await withEnv({ GIT_TRACE: trace }, () => scanRowsBehind(engine, 'default'));
+ expect(scan).toMatchObject({ status: 'known', complete: true, behind: [], pages: 8 });
+ // The eligibility screen reads the index pages' heads from the batch's checkpoint blobs, never one lookup per page.
+ expect(readFileSync(trace, 'utf8').split('\n').filter(line => line.includes('cat-file') && line.includes('--batch-check'))).toEqual([]);
+ } finally { rmSync(trace, { force: true }); }
+ }, { databaseUrl, setup }), 240_000);
+
test('the apply refuses a reimport row whose page moved since the preview and a hold row whose import is no longer refused', () => managedBrain(async ({ engine, ctx, root }) => {
expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
await seedBehind(engine, root, {