Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(repair): url-clone-fences holds a HEAD whose private request page privacy does not read (#7897)

garrytanc40363ff81bd
diff --git a/src/core/repair/url-clone-fences.ts b/src/core/repair/url-clone-fences.ts
index 1807592802..e62d3104fd 100644
--- a/src/core/repair/url-clone-fences.ts
+++ b/src/core/repair/url-clone-fences.ts
@@ -42,6 +42,7 @@ import { createFrontmatterBackup, makeFrontmatterBackupRunId } from '../brain-wr
import { parseFactsFence, renderFactsTable, replaceOrInsertFactsFence, type ParsedFact } from '../facts-fence.ts';
import { extractFactsFromFenceText } from '../facts/extract-from-fence.ts';
import { importFromContent } from '../import-file.ts';
+import { screenImportContent, type ContentRefusal } from '../import-screen.ts';
import { assertSourceFilesystemActive, withSourceFilesystemLock } from '../minions/source-filesystem.ts';
import { OperationError } from '../ops/contract.ts';
import { withPageLock } from '../page-lock.ts';
@@ -111,6 +112,12 @@ async function identityChange(engine: BrainEngine, file: ClonedFile): Promise<st
return null;
}
+/** #7897: why HEAD's bytes would not import as the page (`visibility_noncanonical` and the other import holds), or null. */
+const headHold = (head: string, slug: string): ContentRefusal | null => {
+ const screen = screenImportContent({ content: head, path: `${slug}.md`, holdNoncanonicalPrivate: true });
+ return screen.status === 'refused' ? screen.refusal : null;
+};
+
/** Every changed file of every owned clone in scope, proven or refused. */
async function survey(engine: BrainEngine, sourceIds: string[]): Promise<{ files: ClonedFile[]; listing: RepairListing[] }> {
const files: ClonedFile[] = [];
@@ -139,6 +146,11 @@ async function survey(engine: BrainEngine, sourceIds: string[]): Promise<{ files
continue;
}
const { slug, page_id } = await pageOf(engine, source.id, path);
+ const hold = headHold(head!, slug);
+ if (hold) {
+ listing.push({ item, class: hold.code, detail: `HEAD's bytes would not import as ${slug}, so the file is left as it is: ${hold.message} Fix the file upstream, pull, then preview again: ${preview(source.id)}` });
+ continue;
+ }
files.push({ source_id: source.id, incarnation: source.incarnation, root, path, slug, page_id, head_sha256: sha256(head!), work_sha256: sha256(work!),
rows: proof.added.map(row => row.rowNum), selection: sourceIds });
listing.push({ item, class: 'fence_rows_only', detail: `${proof.added.length} fence row(s) gbrain appended to ${slug}: kept database-only, then the file is restored from HEAD` });
@@ -186,9 +198,12 @@ async function restoreFile(engine: BrainEngine, file: ClonedFile, runId: string)
if (!proof.ok || sha256(head!) !== file.head_sha256 || sha256(work!) !== file.work_sha256) {
return { applied: false, outcome: 'changed_since_preview', detail: { path: file.path } };
}
+ const hold = headHold(head!, file.slug);
+ if (hold) return { applied: false, outcome: 'held', reason: hold.code, detail: { path: file.path, message: hold.message } };
const backup = createFrontmatterBackup(abs, { sourcePath: file.root, backupRoot: gbrainPath('backups', 'url-clone-fences', runId) });
const { moved, inserted } = await keepRowsDatabaseOnly(engine, file, proof.added);
- await importFromContent(engine, file.slug, head!, { sourceId: file.source_id, sourcePath: file.path, sourceRoot: file.root, noEmbed: true });
+ const imported = await importFromContent(engine, file.slug, head!, { sourceId: file.source_id, sourcePath: file.path, sourceRoot: file.root, noEmbed: true, holdNoncanonicalPrivate: true });
+ if (imported.refusal) return { applied: false, outcome: 'held', reason: imported.refusal.code, detail: { path: file.path, message: imported.refusal.message, backup, rows_moved: moved, rows_inserted: inserted } };
assertSourceFilesystemActive();
atomicWriteFileSync(abs, head!);
return { applied: true, outcome: 'restored', detail: { path: file.path, slug: file.slug, backup, rows_moved: moved, rows_inserted: inserted } };
diff --git a/test/e2e/url-clone-fences-private-hold-7897-postgres.test.ts b/test/e2e/url-clone-fences-private-hold-7897-postgres.test.ts
new file mode 100644
index 0000000000..40aeeb9f63
--- /dev/null
+++ b/test/e2e/url-clone-fences-private-hold-7897-postgres.test.ts
@@ -0,0 +1,3 @@
+import { registerPostgresTests } from '../helpers/test-backends.ts';
+
+await registerPostgresTests(() => import('../url-clone-fences-private-hold-7897.test.ts'));
diff --git a/test/url-clone-fences-private-hold-7897.test.ts b/test/url-clone-fences-private-hold-7897.test.ts
new file mode 100644
index 0000000000..f20ff86810
--- /dev/null
+++ b/test/url-clone-fences-private-hold-7897.test.ts
@@ -0,0 +1,138 @@
+/**
+ * #7897: `gbrain repair url-clone-fences` (src/core/repair/url-clone-fences.ts) restored a gbrain-owned clone file by
+ * importing HEAD's bytes through `importFromContent` without the private-request hold and ignored the result. A HEAD
+ * that asks for `visibility: private` in a spelling page privacy does not read (sync holds it `visibility_noncanonical`)
+ * was imported world-readable on an unmanaged brain and reported `restored`.
+ *
+ * Protects: the preview lists such a file under the hold's code and offers nothing to apply; an approval saved before
+ * this release (a preview that listed it `fence_rows_only`) applies as `held` with the code, and the file, its fence
+ * rows and the page stay as they were. HEAD with `visibility: private` still restores a private page, and a HEAD that
+ * asks for nothing private still restores. The brain has no repository of its own (no `sync.repo_path`); the repair
+ * applies only to a source that is an owned Git clone, so there is no clone-less arm.
+ *
+ * PGLite here; Postgres through test/e2e/url-clone-fences-private-hold-7897-postgres.test.ts.
+ */
+import { describe, expect, test } from 'bun:test';
+import { createHash } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { parseFactsFence } from '../src/core/facts-fence.ts';
+import { writeSingleFact } from '../src/core/facts/write-single.ts';
+import { resolveRepairScope, type RepairKind, type RepairResult } from '../src/core/repair/core.ts';
+import { repairRunner } from '../src/core/repair/registry.ts';
+import { saveApprovedSet, type ApprovedSetKey } from '../src/core/persistence/preview-approval.ts';
+import { isPrivatePage } from '../src/core/search/private-visibility.ts';
+import { _resetWriteThroughCacheForTest } from '../src/core/write-through.ts';
+import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const SOURCE = 'source-a';
+const ALICE = 'people/alice-example';
+const FILE = `${ALICE}.md`;
+const KIND = 'url-clone-fences' as RepairKind;
+const person = (extra: string) => `---\ntype: person\ntitle: Alice Example\n${extra}---\n\n# Alice Example\n\nAlice Example works at Acme Example.\n`;
+const git = (dir: string, args: string[]) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', '-C', dir, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+const sha256 = (text: string) => createHash('sha256').update(text).digest('hex');
+
+interface Clone { engine: BrainEngine; root: string; file: () => string; head: () => string; page: () => Promise<{ frontmatter: Record<string, unknown>; compiled_truth: string } | null>;
+ rows: () => Promise<Array<number | null>> }
+
+/**
+ * An owned clone whose first sync imported a world page; HEAD then moves to `extra` frontmatter (a local commit) and an
+ * older gbrain's fence writer appends one row to the working copy, so the file differs from HEAD by fence rows only.
+ */
+async function withClone(databaseUrl: string | undefined, extra: string, run: (c: Clone) => Promise<void>): Promise<void> {
+ const dir = mkdtempSync(join(tmpdir(), 'gbrain-7897-clone-'));
+ try {
+ await withEnv({ GBRAIN_HOME: join(dir, 'home'), GBRAIN_GIT_ALLOW_FILE_TRANSPORT: '1', DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {
+ _resetWriteThroughCacheForTest();
+ const upstream = join(dir, 'upstream');
+ mkdirSync(join(upstream, 'people'), { recursive: true });
+ git(upstream, ['init', '-q', '-b', 'main']);
+ git(upstream, ['config', 'user.email', '[email protected]']);
+ git(upstream, ['config', 'user.name', 'Test Example']);
+ writeFileSync(join(upstream, FILE), person(''));
+ git(upstream, ['add', '-A']);
+ git(upstream, ['commit', '-q', '-m', 'initial']);
+ const root = join(dir, 'clones', SOURCE);
+ execFileSync('git', ['clone', '-q', `file://${upstream}`, root], { stdio: 'pipe' });
+ git(root, ['config', 'user.email', '[email protected]']);
+ git(root, ['config', 'user.name', 'Test Example']);
+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);
+ const config = { remote_url: `file://${upstream}`, managed_clone: true };
+ try {
+ await engine.executeRaw('INSERT INTO sources (id, name, local_path, config) VALUES ($1, $1, $2, $3::text::jsonb)', [SOURCE, root, JSON.stringify(config)]);
+ const { performSync } = await import('../src/commands/sync.ts');
+ expect((await performSync(engine, { sourceId: SOURCE, repoPath: root, noEmbed: true, noExtract: true })).status).toBe('first_sync');
+ writeFileSync(join(root, FILE), person(extra));
+ git(root, ['commit', '-q', '-am', 'head moves']);
+ await engine.executeRaw('UPDATE sources SET config = $1::text::jsonb WHERE id = $2', [JSON.stringify({ remote_url: config.remote_url }), SOURCE]);
+ await writeSingleFact(engine, SOURCE, { fact: 'Alice Example leads the platform team at Acme Example.', provenance: 'test:7897', entity: ALICE, visibility: 'world', remote: false });
+ await engine.executeRaw('UPDATE sources SET config = $1::text::jsonb WHERE id = $2', [JSON.stringify(config), SOURCE]);
+ const c: Clone = {
+ engine, root,
+ file: () => readFileSync(join(root, FILE), 'utf8'),
+ head: () => git(root, ['show', `HEAD:${FILE}`]),
+ page: async () => (await engine.getPage(ALICE, { sourceId: SOURCE })) as never,
+ rows: async () => (await engine.executeRaw<{ row_num: number | null }>('SELECT row_num FROM facts WHERE source_id = $1 AND entity_slug = $2 ORDER BY id', [SOURCE, ALICE]))
+ .map(r => r.row_num === null ? null : Number(r.row_num)),
+ };
+ expect(parseFactsFence(c.file()).facts.length).toBe(1);
+ await run(c);
+ } finally { await close(); }
+ });
+ } finally { rmSync(dir, { recursive: true, force: true }); }
+}
+
+const repair = async (engine: BrainEngine, opts: { apply: boolean; expect?: string }): Promise<RepairResult> =>
+ (await repairRunner(engine, { apply: opts.apply, logger: { info() {}, warn() {}, error() {} } as never }))
+ .run(KIND, await resolveRepairScope(engine, SOURCE), { explicit: true, sourceFlag: SOURCE, ...(opts.expect ? { expect: opts.expect } : {}) });
+
+/** The approved set a preview from before this release saved for the file (it listed it `fence_rows_only`). */
+async function savedApproval(c: Clone): Promise<string> {
+ const [source] = await c.engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id = $1', [SOURCE]);
+ const [page] = await c.engine.executeRaw<{ id: number }>('SELECT id FROM pages WHERE source_id = $1 AND slug = $2', [SOURCE, ALICE]);
+ const hash = 'a'.repeat(64);
+ await saveApprovedSet(c.engine, { command: 'url-clone-fences' as ApprovedSetKey['command'], hash }, [{ source_id: SOURCE, incarnation: source!.incarnation,
+ root: realpathSync(c.root), path: FILE, slug: ALICE, page_id: Number(page!.id), head_sha256: sha256(c.head()), work_sha256: sha256(c.file()),
+ rows: parseFactsFence(c.file()).facts.map(f => f.rowNum), selection: [SOURCE] }]);
+ return hash;
+}
+
+const arms = testBackends().map(backend => ({ backend, databaseUrl: backend === 'pglite' ? undefined : requirePostgresTestDatabase() }));
+for (const { backend, databaseUrl } of arms) describe(`[${backend}] repair url-clone-fences (#7897)`, () => {
+ test('a HEAD asking for Visibility: private previews as held under visibility_noncanonical, with nothing to apply', async () => {
+ await withClone(databaseUrl, 'Visibility: private\n', async c => {
+ const preview = await repair(c.engine, { apply: false });
+ expect({ listing: preview.listing, hash: preview.preview_hash ?? null }).toEqual({ hash: null,
+ listing: [{ item: `${SOURCE}:${FILE}`, class: 'visibility_noncanonical', detail: expect.stringContaining(`gbrain repair url-clone-fences --source ${SOURCE}`) }] });
+ });
+ }, 180_000);
+
+ test('an approval saved before the fix applies as held: nothing is imported, the file and its fence rows stay', async () => {
+ await withClone(databaseUrl, 'visibility: Private\n', async c => {
+ const [file, page, rows] = [c.file(), await c.page(), await c.rows()];
+ const applied = await repair(c.engine, { apply: true, expect: await savedApproval(c) });
+ expect({ applied: applied.applied, outcomes: applied.outcomes, item: applied.outcome_items?.[0] })
+ .toEqual({ applied: 0, outcomes: { held: 1 }, item: expect.objectContaining({ outcome: 'held', reason: 'visibility_noncanonical' }) });
+ expect({ file: c.file() === file, page: await c.page(), rows: await c.rows() }).toEqual({ file: true, page, rows });
+ expect(Object.keys((await c.page())!.frontmatter).map(k => k.toLowerCase())).not.toContain('visibility');
+ });
+ }, 180_000);
+
+ for (const [name, extra, isPrivate] of [['visibility: private', 'visibility: private\n', true], ['no private request', 'tags: [example]\n', false]] as const) {
+ test(`a HEAD with ${name} still restores, and the page is ${isPrivate ? 'private' : 'world-readable'}`, async () => {
+ await withClone(databaseUrl, extra, async c => {
+ const preview = await repair(c.engine, { apply: false });
+ expect(preview.listing!.map(l => l.class)).toEqual(['fence_rows_only']);
+ const applied = await repair(c.engine, { apply: true, expect: preview.preview_hash });
+ expect({ applied: applied.applied, outcomes: applied.outcomes }).toEqual({ applied: 1, outcomes: { restored: 1 } });
+ expect({ file: c.file() === c.head(), rows: await c.rows(), private: isPrivatePage((await c.page())!) }).toEqual({ file: true, rows: [null], private: isPrivate });
+ });
+ }, 180_000);
+ }
+});