Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

test: visibility_noncanonical holds on sync and import, PGLite and Postgres (#7848)

garrytane37157f822e2
diff --git a/test/e2e/visibility-noncanonical-hold-7848-postgres.test.ts b/test/e2e/visibility-noncanonical-hold-7848-postgres.test.ts
new file mode 100644
index 0000000000..c31a5c9062
--- /dev/null
+++ b/test/e2e/visibility-noncanonical-hold-7848-postgres.test.ts
@@ -0,0 +1,3 @@
+import { registerPostgresTests } from '../helpers/test-backends.ts';
+
+await registerPostgresTests(() => import('../visibility-noncanonical-hold-7848.test.ts'));
diff --git a/test/visibility-noncanonical-hold-7848.test.ts b/test/visibility-noncanonical-hold-7848.test.ts
new file mode 100644
index 0000000000..ac791b35c2
--- /dev/null
+++ b/test/visibility-noncanonical-hold-7848.test.ts
@@ -0,0 +1,215 @@
+/**
+ * #7848: a file whose frontmatter asks for `visibility: private` in a spelling page privacy does not read
+ * (`Visibility: private`, `visibility: Private`, a block scalar, a flow list, a merge key, a flow-mapping block) is held
+ * `visibility_noncanonical` by Git sync and `gbrain import`, as `invalid_frontmatter` is, and is never stored
+ * world-readable. #7844 refuses the same request on put_page, edit_page and capture (`visibility_not_applied`); file
+ * ingestion never ran that check, so such a file synced or imported as a world page.
+ *
+ * Protects:
+ * - repo-less `gbrain import` of a plain directory (no Git checkout), unmanaged and managed: each such file is refused
+ * and listed as a failure, no page is written; `visibility: private` (any quoting) and pages that do not ask for
+ * private import unchanged;
+ * - classic (unmanaged) Git sync and managed Git sync: each such file is held `visibility_noncanonical`, the rest of
+ * the source syncs, an existing world page keeps its content, and the file imports private once it says
+ * `visibility: private`;
+ * - doctor `git_held_files` counts the held files and the stored pages whose frontmatter asks for private in a
+ * spelling page privacy does not read (pages written before this release), on a brain with no Git source too.
+ *
+ * PGLite here; Postgres through test/e2e/visibility-noncanonical-hold-7848-postgres.test.ts.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { runImport } from '../src/commands/import.ts';
+import { performSync } from '../src/commands/sync.ts';
+import { importFile } from '../src/core/import-file.ts';
+import { readGitSourceHolds } from '../src/core/persistence/sync-holds.ts';
+import { performManagedSync } from '../src/core/persistence/sync-run.ts';
+import { isPrivatePage } from '../src/core/search/private-visibility.ts';
+import { screenImportContent } from '../src/core/import-screen.ts';
+import { parseMarkdown } from '../src/core/markdown.ts';
+import { noncanonicalPrivateRequest } from '../src/core/persistence/visibility-request.ts';
+import { gitHeldFilesCheck } from '../src/commands/doctor/checks/git-holds.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const backends = testBackends();
+const engines: BrainEngine[] = [];
+let closePostgres: (() => Promise<void>) | undefined;
+const home = mkdtempSync(join(tmpdir(), 'gbrain-7848-'));
+const env = { GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home };
+
+beforeAll(async () => {
+ if (backends.includes('pglite')) { const pglite = new PGLiteEngine(); await pglite.connect({}); await pglite.initSchema(); engines.push(pglite); }
+ if (backends.includes('postgres')) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!); engines.push(pg.engine); closePostgres = pg.close; }
+ for (const each of engines) await each.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+}, 120_000);
+
+afterAll(async () => {
+ for (const each of engines) if (each.kind === 'pglite') await each.disconnect();
+ await closePostgres?.();
+ rmSync(home, { recursive: true, force: true });
+});
+
+const git = (root: string, ...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const commit = (root: string, message: string) => { git(root, 'add', '-A'); git(root, '-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', message); };
+const note = (title: string, extra = '') => `---\ntitle: ${title}\ntype: note\n${extra}---\nA note about acme-example.\n`;
+
+/** Spellings that ask for private but would not store exactly `visibility: private`. */
+const NONCANONICAL: Record<string, string> = {
+ 'notes/upper-key.md': note('Upper key', 'Visibility: private\n'),
+ 'notes/upper-value.md': note('Upper value', 'visibility: Private\n'),
+ 'notes/block-scalar.md': note('Block scalar', 'visibility: |\n private\n'),
+ 'notes/flow-list.md': note('Flow list', 'visibility: [private]\n'),
+ 'notes/merge-key.md': note('Merge key', 'base: &base\n Visibility: Private\n<<: *base\n'),
+ 'notes/flow-mapping.md': '---\n{title: Flow mapping, type: note, Visibility: Private}\n---\nA note about acme-example.\n',
+};
+/** Files that pass unchanged: a canonical private request (any quoting) and pages that do not ask for private. */
+const PRIVATE: Record<string, string> = {
+ 'notes/canonical.md': note('Canonical', 'visibility: private\n'),
+ 'notes/quoted.md': note('Quoted', '"visibility": "private" # keep\n'),
+};
+const WORLD: Record<string, string> = {
+ 'notes/plain.md': note('Plain'),
+ 'notes/world.md': note('World', 'visibility: world\n'),
+};
+const slugOf = (path: string) => path.replace(/\.md$/, '');
+const paths = (files: Record<string, string>) => Object.keys(files).sort();
+const byPath = (a: { path: string }, b: { path: string }) => a.path.localeCompare(b.path);
+
+function writeTree(root: string, files: Record<string, string>): void {
+ mkdirSync(join(root, 'notes'), { recursive: true });
+ for (const [path, content] of Object.entries(files)) writeFileSync(join(root, path), content);
+}
+
+async function newSource(engine: BrainEngine, root: string): Promise<string> {
+ const id = `source-${randomUUID().replace(/-/g, '').slice(0, 12)}`;
+ await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
+ return id;
+}
+
+/** Every noncanonical file left no page, private files are stored private and world files world. */
+async function expectStored(engine: BrainEngine, sourceId: string): Promise<void> {
+ for (const path of paths(NONCANONICAL)) expect({ path, page: await engine.getPage(slugOf(path), { sourceId }) }).toEqual({ path, page: null });
+ for (const path of paths(PRIVATE)) expect({ path, private: isPrivatePage((await engine.getPage(slugOf(path), { sourceId }))!) }).toEqual({ path, private: true });
+ for (const path of paths(WORLD)) expect({ path, private: isPrivatePage((await engine.getPage(slugOf(path), { sourceId }))!) }).toEqual({ path, private: false });
+}
+
+const holdCodes = async (engine: BrainEngine, sourceId: string) =>
+ ((await readGitSourceHolds(engine, { sourceIds: [sourceId] }))[0]?.holds ?? []).map(hold => ({ path: hold.path, code: String(hold.code) })).sort(byPath);
+const heldAs = (code: string, extra: string[] = []) => [...paths(NONCANONICAL), ...extra].map(path => ({ path, code })).sort(byPath);
+
+test('noncanonicalPrivateRequest is true exactly when the content asks for private and the parse would not store visibility: private', () => {
+ const asks = (content: string) => noncanonicalPrivateRequest(content, parseMarkdown(content, 'notes/example.md').frontmatter);
+ for (const [path, content] of Object.entries(NONCANONICAL)) expect({ path, asks: asks(content) }).toEqual({ path, asks: true });
+ for (const [path, content] of Object.entries({ ...PRIVATE, ...WORLD })) expect({ path, asks: asks(content) }).toEqual({ path, asks: false });
+ expect(noncanonicalPrivateRequest('', { Visibility: 'Private' })).toBe(true);
+ expect(noncanonicalPrivateRequest('', { visibility: ['private'] })).toBe(true);
+ expect(noncanonicalPrivateRequest('', { visibility: 'private' })).toBe(false);
+ expect(noncanonicalPrivateRequest('', { visibility: 'world', summary: 'private' })).toBe(false);
+ expect(noncanonicalPrivateRequest('', null)).toBe(false);
+});
+
+test('screenImportContent holds visibility_noncanonical only for file ingestion (holdNoncanonicalPrivate); a page or derived write screen admits it for its own guard', () => {
+ const content = NONCANONICAL['notes/upper-value.md']!;
+ const held = screenImportContent({ content, path: 'notes/upper-value.md', holdNoncanonicalPrivate: true });
+ expect(held.status === 'refused' ? held.refusal.code : held.status).toBe('visibility_noncanonical');
+ expect(screenImportContent({ content, path: 'notes/upper-value.md' }).status).toBe('importable');
+ expect(screenImportContent({ content: PRIVATE['notes/canonical.md']!, path: 'notes/canonical.md', holdNoncanonicalPrivate: true }).status).toBe('importable');
+});
+
+for (const backend of backends) describe(`#7848 non-canonical private requests on file ingestion (${backend})`, () => {
+ const engine = () => engines[backends.indexOf(backend)]!;
+
+ test('repo-less gbrain import of a plain directory refuses each file that asks for private in another spelling; the rest imports unchanged', () => withEnv(env, async () => {
+ const root = mkdtempSync(join(home, 'plain-'));
+ writeTree(root, { ...NONCANONICAL, ...PRIVATE, ...WORLD });
+ const sourceId = await newSource(engine(), root);
+ const result = await runImport(engine(), [root, '--no-embed'], { sourceId });
+ expect(result.imported).toBe(paths(PRIVATE).length + paths(WORLD).length);
+ expect(result.failures.map(failure => failure.path).sort()).toEqual(paths(NONCANONICAL));
+ await expectStored(engine(), sourceId);
+ const direct = await importFile(engine(), join(root, 'notes/upper-value.md'), 'notes/upper-value.md', { noEmbed: true, sourceId });
+ expect({ status: direct.status, code: direct.refusal?.code, skip_reason: direct.skip_reason }).toEqual({ status: 'skipped', code: 'visibility_noncanonical', skip_reason: 'visibility_noncanonical' });
+ expect(await engine().getPage('notes/upper-value', { sourceId })).toBeNull();
+ }), 180_000);
+
+ test('classic Git sync holds each such file visibility_noncanonical, keeps an existing world page, and imports it private once fixed', () => withEnv(env, async () => {
+ const root = mkdtempSync(join(home, 'classic-'));
+ git(root, 'init', '-q');
+ writeTree(root, { 'notes/turned.md': note('Turned') });
+ commit(root, 'seed');
+ const sourceId = await newSource(engine(), root);
+ const sync = () => performSync(engine(), { repoPath: root, sourceId, noPull: true, noEmbed: true, noExtract: true, concurrency: 1 });
+ await sync();
+ const before = (await engine().getPage('notes/turned', { sourceId }))!;
+ expect(isPrivatePage(before)).toBe(false);
+ writeTree(root, { ...NONCANONICAL, ...PRIVATE, ...WORLD, 'notes/turned.md': note('Turned', 'Visibility: Private\n') });
+ commit(root, 'ask for private');
+ await sync();
+ expect(await holdCodes(engine(), sourceId)).toEqual(heldAs('visibility_noncanonical', ['notes/turned.md']));
+ await expectStored(engine(), sourceId);
+ const kept = (await engine().getPage('notes/turned', { sourceId }))!;
+ expect({ hash: kept.content_hash, private: isPrivatePage(kept) }).toEqual({ hash: before.content_hash, private: false });
+ writeTree(root, { 'notes/turned.md': note('Turned', 'visibility: private\n') });
+ commit(root, 'canonical private');
+ await sync();
+ expect(await holdCodes(engine(), sourceId)).toEqual(heldAs('visibility_noncanonical'));
+ expect(isPrivatePage((await engine().getPage('notes/turned', { sourceId }))!)).toBe(true);
+ const doctor = await gitHeldFilesCheck(engine(), [sourceId]);
+ expect(doctor.details?.visibility_noncanonical).toEqual({ held: paths(NONCANONICAL).length, stored: 0 });
+ }), 180_000);
+
+ test('doctor git_held_files counts stored pages that ask for private in another spelling, on a brain with no Git source', () => withEnv(env, async () => {
+ const sourceId = await newSource(engine(), mkdtempSync(join(home, 'stored-')));
+ const put = (slug: string, frontmatter: Record<string, unknown>) =>
+ engine().putPage(slug, { type: 'note', title: slug, compiled_truth: 'A private note about acme-example.', frontmatter }, { sourceId });
+ await put('notes/legacy-upper', { Visibility: 'Private' });
+ await put('notes/legacy-list', { visibility: ['private'] });
+ await put('notes/legacy-canonical', { visibility: 'private' });
+ await put('notes/legacy-world', { visibility: 'world', summary: 'private' });
+ const doctor = await gitHeldFilesCheck(engine(), [sourceId]);
+ expect(doctor.details?.visibility_noncanonical).toEqual({ held: 0, stored: 2 });
+ expect(doctor.status).toBe('warn');
+ expect(doctor.message).toContain('2 stored page(s)');
+ }), 180_000);
+});
+
+describe('#7848 managed brains', () => {
+ for (const backend of backends) {
+ const databaseUrl = backend === 'postgres' ? process.env.DATABASE_URL : undefined;
+ const OPTS = { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true };
+
+ test(`[${backend}] managed Git sync holds each such file visibility_noncanonical, the rest syncs, and a fixed file imports private`, () => managedBrain(async ({ engine, root }) => {
+ writeTree(root, { ...PRIVATE, ...WORLD });
+ commit(root, 'seed');
+ expect((await performManagedSync(engine, OPTS)).status).toBe('first_sync');
+ writeTree(root, NONCANONICAL);
+ commit(root, 'ask for private');
+ const result = await performManagedSync(engine, OPTS);
+ expect((result.held ?? []).map(hold => ({ path: hold.path, code: String(hold.code) })).sort(byPath)).toEqual(heldAs('visibility_noncanonical'));
+ expect(await holdCodes(engine, 'default')).toEqual(heldAs('visibility_noncanonical'));
+ await expectStored(engine, 'default');
+ writeTree(root, { 'notes/upper-key.md': note('Upper key', 'visibility: private\n') });
+ commit(root, 'canonical private');
+ await performManagedSync(engine, OPTS);
+ expect((await holdCodes(engine, 'default')).map(hold => hold.path)).not.toContain('notes/upper-key.md');
+ expect(isPrivatePage((await engine.getPage('notes/upper-key', { sourceId: 'default' }))!)).toBe(true);
+ }, { ...(databaseUrl ? { databaseUrl } : {}), setup: ({ root }) => { git(root, 'init', '-q'); } }), 180_000);
+
+ test(`[${backend}] managed gbrain import of a directory with no Git checkout refuses each such file; the rest imports unchanged`, () => managedBrain(async ({ engine, root }) => {
+ writeTree(root, { ...NONCANONICAL, ...PRIVATE, ...WORLD });
+ const result = await runImport(engine, [root, '--no-embed'], { sourceId: 'default' });
+ expect(result.failures.map(failure => failure.path).sort()).toEqual(paths(NONCANONICAL));
+ for (const failure of result.failures) expect(failure.error).toContain('visibility: private');
+ await expectStored(engine, 'default');
+ }, databaseUrl ? { databaseUrl } : {}), 180_000);
+ }
+});