Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: adversarial review fixes

garrytan541e1dae67ee
diff --git a/src/commands/sources-canonical.ts b/src/commands/sources-canonical.ts
index 46ae757151..b6bbd912e5 100644
--- a/src/commands/sources-canonical.ts
+++ b/src/commands/sources-canonical.ts
@@ -30,7 +30,7 @@ export function parseCanonicalArgs(args: string[]): { sourceId: string; to: 'git
const arg = args[i]!;
if (arg === '--to') { to = args[++i] ?? null; continue; }
if (arg === '--expect') { expect = args[++i] ?? null; if (!expect) throw new OperationError('invalid_params', '--expect needs the preview hash.', USAGE); continue; }
- if (arg in files) { const value = args[++i]; if (!value || value.startsWith('-')) throw new OperationError('invalid_params', `${arg} needs a file path.`, USAGE); files[arg as keyof typeof files] = value; continue; }
+ if (Object.hasOwn(files, arg)) { const value = args[++i]; if (!value || value.startsWith('-')) throw new OperationError('invalid_params', `${arg} needs a file path.`, USAGE); files[arg as keyof typeof files] = value; continue; }
if (arg === '--apply') { apply = true; continue; }
if (arg === '--dry-run' || arg === '--json' || arg === '--no-embed' || arg === '--no-extract') continue;
if (arg.startsWith('-')) throw new OperationError('invalid_params', `Unknown sources canonical flag: ${arg}.`, USAGE);
diff --git a/src/core/persistence/git-conversion.ts b/src/core/persistence/git-conversion.ts
index b16aa5f578..4f3cbc54d8 100644
--- a/src/core/persistence/git-conversion.ts
+++ b/src/core/persistence/git-conversion.ts
@@ -40,7 +40,7 @@
* remaining categories are reported. `--to mixed` removes the flag and changes nothing else.
*/
import { chmodSync, lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
-import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts, SyncResult } from '../../commands/sync.ts';
import type { Action } from '../agent-output.ts';
@@ -213,7 +213,7 @@ export async function assertOutsideSourceRoots(engine: BrainEngine, file: string
try { directory = realpathSync.native(dirname(target)); } catch {
throw opError('invalid_params', `${flag} ${file}: its directory does not exist.`, `Pass a path in an existing directory outside every source's checkout, e.g. under ~/.gbrain.`);
}
- const real = join(directory, target.slice(dirname(target).length + 1));
+ const real = join(directory, basename(target));
let linked = real;
try { linked = realpathSync.native(real); } catch { /* not written yet */ }
const roots = await engine.executeRaw<{ id: string; local_path: string }>('SELECT id, local_path FROM sources WHERE local_path IS NOT NULL');
@@ -237,7 +237,9 @@ export async function writeConversionFile(engine: BrainEngine, file: string, fla
try { if (lstatSync(real).isSymbolicLink()) throw opError('invalid_params', `${flag} ${file} is a symlink, so it was not written.`, 'Pass the path of a regular file.'); }
catch (error) { if (error instanceof OperationError) throw error; }
const temp = `${real}.${process.pid}.tmp`;
- try { writeFileSync(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' }); chmodSync(temp, 0o600); renameSync(temp, real); } finally { rmSync(temp, { force: true }); }
+ try { writeFileSync(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o600, flag: 'wx' }); chmodSync(temp, 0o600); renameSync(temp, real); }
+ catch (error) { throw opError('invalid_params', `${flag} ${file} could not be written (${(error as NodeJS.ErrnoException).code ?? 'error'}).`, 'Pass a writable path outside every source checkout, e.g. under ~/.gbrain.'); }
+ finally { rmSync(temp, { force: true }); }
return real;
}
@@ -714,7 +716,7 @@ function approvalItems(plan: GitConversionPlan): Record<string, string> {
for (const item of plan.adopt) items[`adopt:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.blob ?? null, item.evidence ?? null, item.state, item.disk]);
for (const item of plan.gitWins) items[`git_wins:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.blob, item.evidence ?? null, item.reason]);
for (const item of plan.withhold) items[`withhold:${item.path}`] = JSON.stringify([item.id, item.slug, item.revision, item.kind, item.declared, item.disk]);
- for (const hold of plan.holds) items[`hold:${hold.path}`] = JSON.stringify([hold.code, hold.side, hold.observed_at, hold.action]);
+ for (const hold of plan.holds) items[`hold:${hold.path}`] = JSON.stringify([hold.code, hold.side, hold.action]);
for (const item of plan.parked) items[`parked:${item.effect}`] = JSON.stringify([item.action, item.codes]);
return items;
}
@@ -738,7 +740,7 @@ export async function readConversionApproval(engine: BrainEngine, file: string,
const bad = (why: string) => opError('invalid_params', `--approved ${file}: ${why}`, `Write the approval with gbrain sources canonical ${sourceId} --to git --dry-run --approval-out <file> and pass it unchanged.`);
let parsed: ConversionApproval;
try { parsed = JSON.parse(readFileSync(real, 'utf8')); } catch { throw bad('it is not JSON.'); }
- if (parsed?.kind !== APPROVAL_KIND || parsed.version !== 1 || typeof parsed.items !== 'object' || !Array.isArray(parsed.decisions)) throw bad(`it is not a ${APPROVAL_KIND} file (version 1).`);
+ if (parsed?.kind !== APPROVAL_KIND || parsed.version !== 1 || typeof parsed.items !== 'object' || parsed.items === null || Array.isArray(parsed.items) || !Array.isArray(parsed.decisions)) throw bad(`it is not a ${APPROVAL_KIND} file (version 1).`);
if (parsed.source_id !== sourceId) throw bad(`it is for source ${String(parsed.source_id)}, not ${sourceId}.`);
return { ...parsed, decisions: decisionEntries(parsed.decisions, bad) };
}
diff --git a/src/core/persistence/git-gap.ts b/src/core/persistence/git-gap.ts
index 75a8fab142..1d4f346c32 100644
--- a/src/core/persistence/git-gap.ts
+++ b/src/core/persistence/git-gap.ts
@@ -392,8 +392,11 @@ export function decideDirection(row: string, head: string | null, versions: Read
return { kind: 'no_evidence', basis: 'no_shared_state', blob: null, version: null };
}
-type VersionRow = { id: number; page_id: string; compiled_truth: string | null; frontmatter: unknown; timeline: string | null; title: string | null; type: string | null; tags: unknown;
- segmented: boolean; is_deleted: boolean | null };
+type VersionRow = { id: number; page_id: string; compiled_truth: string; frontmatter: unknown; timeline: string | null; title: string | null; type: string | null; tags: unknown };
+/** A version the evidence read lists: `readable` when it is an inline, live snapshot with a body (its digest can be computed). */
+type VersionMeta = { id: number | string; page_id: string; bytes: number | string; readable: boolean };
+/** At most this many version body bytes are held at once (a larger version is read alone). */
+const VERSION_READ_BYTES = 32 * 1024 * 1024;
const json = <T>(value: unknown, fallback: T): T => value == null ? fallback : typeof value === 'string' ? JSON.parse(value) as T : value as T;
/**
@@ -423,7 +426,9 @@ export async function readDirectionEvidence(engine: BrainEngine, input: { source
const history = await readPathHistory(input.gitRoot, tip, scope, new Set([...paths.values()].flatMap(set => [...set])));
const none = (basis: EvidenceBasis, committed_at: number | null = null): Evidence => ({ kind: 'no_evidence', basis, blob: null, version: null, versions: 0, blobs: 0, committed_at });
if (!history) { for (const subject of subjects) out.set(subject.id, none('history_unreadable')); return out; }
- const older = (subject: EvidenceSubject) => {
+ const olderOf = new Map<string, { list: HistoryEntry[]; committed_at: number | null; capped: boolean }>();
+ const older = (subject: EvidenceSubject) => olderOf.get(subject.id) ?? olderOf.set(subject.id, readOlder(subject)).get(subject.id)!;
+ const readOlder = (subject: EvidenceSubject) => {
const merged = [...paths.get(subject.id)!].flatMap(path => history.get(path) ?? []).sort((a, b) => a.seq - b.seq);
const seen = new Set<string>(), list: HistoryEntry[] = [];
let pastTip = false;
@@ -437,6 +442,8 @@ export async function readDirectionEvidence(engine: BrainEngine, input: { source
const withdrawals = await readExportWithdrawals(engine, sourceId);
for (let start = 0; start < subjects.length; start += EVIDENCE_BATCH) {
const batch = subjects.slice(start, start + EVIDENCE_BATCH);
+ const segmentedSizes = new Map<string, Map<string, number> | null>();
+ for (const subject of batch) if (subject.segmented) segmentedSizes.set(subject.id, await readBlobSizes(root, older(subject).list.map(entry => entry.oid)));
const read = await engine.transaction(async tx => {
await tx.executeRaw('SET TRANSACTION READ ONLY');
const rows = await tx.executeRaw<{ id: string; revision: string | null; gap: boolean }>(`SELECT id::text, knowledge_revision::text AS revision, version_history_gap_at IS NOT NULL AS gap
@@ -444,19 +451,34 @@ export async function readDirectionEvidence(engine: BrainEngine, input: { source
const live = new Map(rows.map(row => [row.id, row.revision])), gaps = new Set(rows.filter(row => row.gap).map(row => row.id));
const current = batch.filter(subject => live.has(subject.id) && live.get(subject.id) === subject.revision);
const snapshots = await readExportPages(tx, current.map(subject => ({ id: subject.id, source_id: sourceId, slug: subject.slug })), withdrawals);
- const versions = current.length ? await tx.executeRaw<VersionRow>(`SELECT id, page_id::text AS page_id, compiled_truth, frontmatter, timeline, title, type, tags,
- COALESCE(body_storage,'inline')='segmented' AS segmented, is_deleted FROM (SELECT v.*, row_number() OVER (PARTITION BY v.page_id ORDER BY v.id DESC) AS rank FROM page_versions v
- WHERE v.page_id = ANY(string_to_array($1::text, ',')::int[])) ranked WHERE rank <= $2 ORDER BY page_id, id DESC`, [current.map(subject => subject.id).join(','), EVIDENCE_DEPTH]) : [];
+ const versions = current.length ? await tx.executeRaw<VersionMeta>(`SELECT id, page_id::text AS page_id, COALESCE(octet_length(compiled_truth), 0) + COALESCE(octet_length(timeline), 0) AS bytes,
+ (COALESCE(body_storage,'inline')='inline' AND NOT COALESCE(is_deleted, false) AND compiled_truth IS NOT NULL) AS readable FROM (SELECT v.id, v.page_id, v.compiled_truth, v.timeline, v.body_storage,
+ v.is_deleted, row_number() OVER (PARTITION BY v.page_id ORDER BY v.id DESC) AS rank FROM page_versions v WHERE v.page_id = ANY(string_to_array($1::text, ',')::int[])) ranked
+ WHERE rank <= $2 ORDER BY page_id, id DESC`, [current.map(subject => subject.id).join(','), EVIDENCE_DEPTH]) : [];
+ const snapshotOf = new Map(current.map((subject, i) => [subject.id, snapshots[i]!]));
+ const digests = new Map<string, string>(), wanted = versions.filter(row => row.readable);
+ for (let from = 0; from < wanted.length;) {
+ let to = from, bytes = 0;
+ while (to < wanted.length && (to === from || bytes + Number(wanted[to]!.bytes) <= VERSION_READ_BYTES)) bytes += Number(wanted[to++]!.bytes);
+ const rows = await tx.executeRaw<VersionRow>(`SELECT id, page_id::text AS page_id, compiled_truth, frontmatter, timeline, title, type, tags FROM page_versions
+ WHERE id = ANY(string_to_array($1::text, ',')::int[])`, [wanted.slice(from, to).map(row => row.id).join(',')]);
+ for (const row of rows) {
+ const page = snapshotOf.get(row.page_id)!.page;
+ digests.set(String(row.id), digest(reconcileCanonical({ type: (row.type ?? page.type) as Page['type'], title: row.title ?? page.title, compiled_truth: row.compiled_truth,
+ timeline: row.timeline ?? '', frontmatter: json(row.frontmatter, {}) }, json<string[]>(row.tags, []))));
+ }
+ from = to;
+ }
const segmentedOids = new Map<string, Set<string>>();
for (const [i, subject] of current.entries()) {
if (!subject.segmented) continue;
- const sizes = await readBlobSizes(root, older(subject).list.map(entry => entry.oid));
+ const sizes = segmentedSizes.get(subject.id);
segmentedOids.set(subject.id, sizes ? await segmentedExportBlobIds(tx, snapshots[i]!, withdrawals, new Set(sizes.values()), format) : new Set());
}
- return { current, snapshots, versions, segmentedOids, gaps };
+ return { current, snapshots, versions, digests, segmentedOids, gaps };
});
for (const subject of batch) if (!read.current.includes(subject)) out.set(subject.id, none('changed_during_read', older(subject).committed_at));
- const byPage = new Map<string, VersionRow[]>();
+ const byPage = new Map<string, VersionMeta[]>();
for (const row of read.versions) byPage.set(row.page_id, [...byPage.get(row.page_id) ?? [], row]);
const inline = read.current.filter(subject => !subject.segmented);
const oids = [...new Set(inline.flatMap(subject => [subject.blob.oid, ...older(subject).list.map(entry => entry.oid)]))];
@@ -490,11 +512,11 @@ export async function readDirectionEvidence(engine: BrainEngine, input: { source
continue;
}
const snapshot = read.snapshots[i]!;
- const versions = (byPage.get(subject.id) ?? []).filter(row => !row.segmented && !row.is_deleted && row.compiled_truth !== null).map(row => ({ id: Number(row.id),
- digest: digest(reconcileCanonical({ type: (row.type ?? snapshot.page.type) as Page['type'], title: row.title ?? snapshot.page.title, compiled_truth: row.compiled_truth!,
- timeline: row.timeline ?? '', frontmatter: json(row.frontmatter, {}) }, json<string[]>(row.tags, []))) }));
+ const rows = byPage.get(subject.id) ?? [];
+ const versions = rows.flatMap(row => read.digests.has(String(row.id)) ? [{ id: Number(row.id), digest: read.digests.get(String(row.id))! }] : []);
+ const unread = read.gaps.has(subject.id) || rows.length >= EVIDENCE_DEPTH || versions.length < rows.length;
const decided = decideDirection(digest(reconcileCanonical(snapshot.page, snapshot.tags)), fileDigests.get(`${subject.id}:${subject.blob.oid}`) ?? null, versions,
- list.map(entry => ({ oid: entry.oid, digest: fileDigests.get(`${subject.id}:${entry.oid}`) ?? null })), { versions: read.gaps.has(subject.id) || (byPage.get(subject.id)?.length ?? 0) >= EVIDENCE_DEPTH, blobs: capped });
+ list.map(entry => ({ oid: entry.oid, digest: fileDigests.get(`${subject.id}:${entry.oid}`) ?? null })), { versions: unread, blobs: capped });
out.set(subject.id, { ...decided, versions: versions.length, blobs: list.length, committed_at });
}
}
diff --git a/test/git-canonical-conversion-evidence-d4.test.ts b/test/git-canonical-conversion-evidence-d4.test.ts
index 6ad63105a2..11d9d032a2 100644
--- a/test/git-canonical-conversion-evidence-d4.test.ts
+++ b/test/git-canonical-conversion-evidence-d4.test.ts
@@ -276,6 +276,12 @@ for (const backend of testBackends()) describe(`[${backend}] conversion by conte
const twice = join(files.dir, 'twice.json');
await writeConversionFile(engine, twice, '--approval-out', { ...approval, decisions: [...approval.decisions, approval.decisions[0]] });
await expect(readConversionApproval(engine, twice, 'default')).rejects.toMatchObject({ code: 'invalid_params', message: expect.stringContaining('decided twice') });
+ const noItems = join(files.dir, 'no-items.json');
+ await writeConversionFile(engine, noItems, '--approval-out', { ...approval, items: null });
+ await expect(readConversionApproval(engine, noItems, 'default')).rejects.toMatchObject({ code: 'invalid_params' });
+ writeFileSync(`${join(files.dir, 'stale.json')}.${process.pid}.tmp`, '');
+ await expect(writeConversionFile(engine, join(files.dir, 'stale.json'), '--approval-out', approval)).rejects.toMatchObject({ code: 'invalid_params' });
+ expect(await cli(engine, ['constructor', '--to', 'git', '--dry-run'])).not.toMatchObject({ message: expect.stringContaining('needs a file path') });
writeFileSync(join(root, 'notes/db.md'), note('notes/db', 'A commit that landed after the approval.'));
git(root, 'commit', '-qam', 'a commit after the approval');
@@ -296,8 +302,13 @@ for (const backend of testBackends()) describe(`[${backend}] conversion by conte
try {
await evidenceSource(brain);
const decisions = await decide(brain, files.dir, { 'notes/both': 'git_wins', 'notes/none': 'adopt' });
+ const incarnation = (await planGitConversion(engine, 'default')).incarnation;
+ await staleHold(engine, incarnation, 'notes/same');
const approval = conversionApproval(await planGitConversion(engine, 'default', { decisions }));
- expect(await applyApprovedConversion(engine, 'default', approval, PROCESSING)).toMatchObject({ status: 'converted', committed: { pages: 2 } });
+ // A sync re-screening a held path rewrites only its observed_at: the approval still applies whole.
+ await new Promise(resolve => setTimeout(resolve, 5));
+ await staleHold(engine, incarnation, 'notes/same');
+ expect(await applyApprovedConversion(engine, 'default', approval, PROCESSING)).toMatchObject({ status: 'converted', committed: { pages: 2 }, holds_cleared: 1 });
expect(await strict(engine)).toBe(true);
} finally { files.done(); }
}, { databaseUrl, setup }), 240_000);
diff --git a/test/git-gap-direction-evidence-d4.test.ts b/test/git-gap-direction-evidence-d4.test.ts
index d6d3a36d39..d6b587942f 100644
--- a/test/git-gap-direction-evidence-d4.test.ts
+++ b/test/git-gap-direction-evidence-d4.test.ts
@@ -226,6 +226,17 @@ for (const backend of testBackends()) describe(`git-gap direction evidence (d4):
expect(pages.find(page => page.slug === 'notes/g')).toMatchObject({ evidence: { kind: 'both_moved', basis: 'conflicting_history' } });
}, 120000);
+ test('a version with no comparable content (a deletion, a segmented snapshot) counts as unread: no one-sided Git verdict', async () => {
+ const root = await fixture();
+ const [page] = await engine.executeRaw<{ id: string }>("SELECT id::text FROM pages WHERE slug='notes/file-moved'");
+ await engine.createVersion('notes/file-moved');
+ await engine.executeRaw("INSERT INTO page_versions (page_id, compiled_truth, frontmatter, is_deleted) VALUES ($1, 'A deleted state.', '{}', true)", [page!.id]);
+ const pages: GitGapPage[] = [];
+ await censusGitGap(engine, 'default', { sample: 0, evidence: true, collect: entry => { pages.push(entry); } });
+ expect(pages.find(entry => entry.slug === 'notes/file-moved')).toMatchObject({ evidence: { kind: 'no_evidence', basis: 'history_truncated' } });
+ expect(root).not.toBe('');
+ }, 120000);
+
test('a pruned version history counts as unread: an inline page with a history gap gets no one-sided Git verdict', async () => {
await fixture();
await engine.executeRaw("UPDATE pages SET version_history_gap_at = now() WHERE slug = 'notes/file-moved'");