Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: pre-landing review fixes (#7848, #7850, #7871)

garrytan748bf63162fc
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f5ae4d72b..a92102d199 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,10 +40,11 @@ gbrain dream --dry-run --phase synthesize # preview now records nothing
- Sessions an earlier release already froze, and pages or atoms already written into the wrong source, are not changed or moved. Repair stays a separate, preview-first step.
- Held sessions and held generation members are counted in the phase summary and `details.source_routing`, with the command that releases them.
+- A synthesis job queued by an earlier release is checked again only on a brain with managed persistence, where the job records its transcript. On other brains, let the waiting dream jobs (`gbrain jobs list --status waiting`) finish before you add, archive or recreate sources.
### Private files in another spelling
-**Git sync and `gbrain import` hold a file that asks for `visibility: private` in a spelling privacy does not read, instead of publishing it world-readable (#7848).** Page privacy reads only the exact lowercase `visibility: private`. A file that wrote `Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key used to sync or import as a page anyone could read. Now classic and managed sync hold it as `visibility_noncanonical` (the rest of the source syncs, and an existing page keeps its last imported version), and `gbrain import`, with or without a Git checkout, refuses it and imports the rest. Write `visibility: private` on its own frontmatter line and commit, or import the file again. `gbrain doctor` (`git_held_files`) counts the held files and the stored pages that already ask for private this way; nothing rewrites them.
+**Git sync and `gbrain import` hold a file that asks for `visibility: private` in a spelling privacy does not read, instead of publishing it world-readable (#7848).** Page privacy reads only the exact lowercase `visibility: private`. A file that wrote `Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key used to sync or import as a page anyone could read. Now classic and managed sync hold it as `visibility_noncanonical` (the rest of the source syncs, and an existing page keeps its last imported version), and `gbrain import`, with or without a Git checkout, refuses it and imports the rest. Write `visibility: private` on its own frontmatter line and commit, or import the file again. `gbrain doctor` (`git_held_files`) counts the held files and the stored pages that already ask for private this way; nothing rewrites them. On a brain without managed persistence, `gbrain put --file` and `gbrain capture --file` of a page over the single-string ceiling now refuse such a request with `visibility_not_applied`, as a smaller write does.
### Itemized changes
diff --git a/src/cli/commands/append.ts b/src/cli/commands/append.ts
index f4986aab76..1317f06f38 100644
--- a/src/cli/commands/append.ts
+++ b/src/cli/commands/append.ts
@@ -456,7 +456,7 @@ async function streamLocalPage(connect: () => Promise<BrainEngine>, path: string
assertPageRevision(await engine.readPageSnapshot(slug, { sourceId, includeDeleted: true }), engineMutationPrecondition(parseMutationPrecondition(params)));
const result = await importStreamedMarkdownFile(engine, resolve(path), relativePath, { sourceId, inferFrontmatter: false, content }, stat, policy);
if (result.status === 'skipped' && result.error) {
- throw opError(result.refusal?.code === 'invalid_frontmatter' ? 'invalid_frontmatter' : 'invalid_params', result.error,
+ throw opError(result.refusal?.code === 'invalid_frontmatter' ? 'invalid_frontmatter' : result.refusal?.code === 'visibility_noncanonical' ? 'visibility_not_applied' : 'invalid_params', result.error,
`${path} was not written as ${slug}: fix what the message names (or split the file into smaller pages) and run the command again.`);
}
return result;
diff --git a/src/commands/doctor/checks/git-holds.ts b/src/commands/doctor/checks/git-holds.ts
index db8f71ef9f..00085d4a59 100644
--- a/src/commands/doctor/checks/git-holds.ts
+++ b/src/commands/doctor/checks/git-holds.ts
@@ -121,7 +121,7 @@ async function readNoncanonicalPrivate(engine: BrainEngine, sourceIds?: string[]
const candidates = await engine.executeRaw<{ source_id: string; slug: string; frontmatter: Record<string, unknown> | string | null }>(`SELECT p.source_id, p.slug, p.frontmatter
FROM pages p JOIN sources s ON s.id=p.source_id
WHERE p.deleted_at IS NULL AND s.archived IS NOT TRUE AND ($1::text[] IS NULL OR p.source_id=ANY($1::text[])) AND jsonb_typeof(p.frontmatter)='object'
- AND p.frontmatter::text ILIKE '%visibility%' AND p.frontmatter::text ILIKE '%private%' AND p.frontmatter->>'visibility' IS DISTINCT FROM 'private'
+ AND EXISTS (SELECT 1 FROM jsonb_object_keys(p.frontmatter) k WHERE lower(k)='visibility') AND p.frontmatter->>'visibility' IS DISTINCT FROM 'private'
ORDER BY p.source_id, p.slug`, [ids]);
const stored = candidates.filter(row => noncanonicalPrivateRequest('', typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter));
return { held: Number(heldRow?.n ?? 0), stored: stored.length, first: stored.slice(0, SAMPLE_PATHS).map(({ source_id, slug }) => ({ source_id, slug })) };
diff --git a/src/core/behavior-change-notice.ts b/src/core/behavior-change-notice.ts
index 38d238253d..d616610f20 100644
--- a/src/core/behavior-change-notice.ts
+++ b/src/core/behavior-change-notice.ts
@@ -1330,11 +1330,11 @@ export const BEHAVIOR_CHANGES: ReadonlyArray<{ since: string; text: ChangeText;
{ since: '0.60.482.0', id: 'extract_atoms_session_source_7846', text: 'Dream `extract_atoms` writes a captured session\'s atoms only into the source its session recorded (the `.src-<id>` name in the corpus `sourced/` folder, or the session\'s frozen record), never into `default`. Each source\'s run now reads the session corpus and keeps only its own sessions (`gbrain dream --source <id> --phase extract_atoms`); meeting transcripts stay with `default`. A session whose source never resolved, and a file written before sessions recorded their source on a brain with more than one source (already-swept ones included), is held until `gbrain sweep --assign-corpus <source>` previews and `--apply` maps it. The phase summary and `details.source_routing` count both. Atoms an earlier release already wrote into the wrong source are not moved (#7846).' },
// #7868, #7850, #7869, #7870, #7871 frozen-source privacy wave. `since` is stamped at the merge slot.
{ since: 'TBD', id: 'corpus_session_tier4_only_7868', text: 'A captured session whose source was never recorded is no longer filed under `sources.default`, the only non-default source or `default` when the brain resolves it later: only a registered source whose folder contains the session\'s working directory counts. Any other session stays held as `corpus_source_unresolved` until `gbrain sweep --assign-corpus <source>` previews and `--apply` maps it. A brain with one source still uses that source, without recording it, so a source added later holds the session instead. Sessions an earlier release already recorded keep their source (#7868).' },
- { since: 'TBD', id: 'synthesize_session_recheck_7850', text: 'A dream synthesis job checks its session\'s source again when it runs and is refused (`session_source_changed`) before any model call when the session now belongs to another source or its source was deleted and recreated; jobs queued by an earlier release are checked too. `gbrain dream --dry-run` no longer records a session\'s source while previewing synthesize or extract_atoms (#7850).' },
+ { since: 'TBD', id: 'synthesize_session_recheck_7850', text: 'A dream synthesis job checks its session\'s source again when it runs and is refused (`session_source_changed`) before any model call when the session now belongs to another source or its source was deleted and recreated; on a managed brain, jobs queued by an earlier release are checked too. `gbrain dream --dry-run` no longer records a session\'s source while previewing synthesize or extract_atoms (#7850).' },
{ since: 'TBD', id: 'atom_retry_resume_session_source_7869_7870', text: 'An explicit atom retry (`gbrain jobs submit extract-atoms-drain` with `retryRequestId`) of a captured session that does not belong to the retry\'s source is refused with `transcript_source_mismatch` and starts nothing. A crashed atom generation whose pending members came from such a session is not resumed: the members are held, their atom pages left as they are, and counted in `details.source_routing.generation_members_held` (#7869, #7870).' },
{ since: 'TBD', id: 'corpus_realpath_7871', text: 'Dream synthesis and extract_atoms recognize a session-corpus file through symlinks (a meeting folder or corpus path that points into the corpus, a linked `sourced/` folder), so such a file is routed to its session\'s source instead of being read as an ordinary transcript; a listed file whose real path cannot be read is held as `corpus_path_unresolvable` (#7871).' },
// #7848 non-canonical private requests on file ingestion. `since` is stamped at the merge slot.
- { since: 'TBD', id: 'visibility_noncanonical_hold_7848', text: 'Git sync (classic and managed) and `gbrain import` (with or without a Git checkout) hold a file whose frontmatter asks for `visibility: private` in a spelling page privacy does not read (`Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key) as `visibility_noncanonical` instead of storing a world-readable page; the rest of the source syncs and an existing page keeps its last imported version. Write `visibility: private` on its own frontmatter line (lowercase) and commit, or import the file again. Doctor `git_held_files` counts the held files and the stored pages that already ask for private this way (`details.visibility_noncanonical`); nothing rewrites them.' },
+ { since: 'TBD', id: 'visibility_noncanonical_hold_7848', text: 'Git sync (classic and managed) and `gbrain import` (with or without a Git checkout) hold a file whose frontmatter asks for `visibility: private` in a spelling page privacy does not read (`Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key) as `visibility_noncanonical` instead of storing a world-readable page; the rest of the source syncs and an existing page keeps its last imported version. Write `visibility: private` on its own frontmatter line (lowercase) and commit, or import the file again. Doctor `git_held_files` counts the held files and the stored pages that already ask for private this way (`details.visibility_noncanonical`); nothing rewrites them. On a brain without managed persistence, `gbrain put --file` and `gbrain capture --file` of a page over the single-string ceiling refuse such a request with `visibility_not_applied`, as a smaller write does.' },
];
/** The newest disclosed change's release: the notice id moves only when a release adds rows. */
diff --git a/src/core/cycle/synthesize-source-routing.ts b/src/core/cycle/synthesize-source-routing.ts
index 7f675b0252..1ce5f6e42b 100644
--- a/src/core/cycle/synthesize-source-routing.ts
+++ b/src/core/cycle/synthesize-source-routing.ts
@@ -26,7 +26,7 @@
* (`transcriptSourceHold`, #7869, #7870). Dry runs resolve read-only.
*/
-import { realpathSync } from 'node:fs';
+import { lstatSync, realpathSync } from 'node:fs';
import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { PhaseResult } from '../cycle.ts';
@@ -66,6 +66,7 @@ function corpusLocation(filePath: string, corpusDir: string): CorpusFileLocation
if (realRoot === null) return lexical;
const realSpool = realpathOrMissing(corpusSpoolDir(root));
const realFile = realpathOrMissing(filePath);
+ if (realFile === null && !lstatSync(filePath, { throwIfNoEntry: false })) return lexical;
if (realRoot === undefined || realSpool === undefined || !realFile) return 'unresolvable';
const real = locate(realFile, realRoot, realSpool);
return lexical === 'spool' || real === 'spool' ? 'spool' : lexical ?? real;
@@ -119,7 +120,8 @@ export async function corpusSourceChanged(engine: BrainEngine, pin: { raw_source
/**
* A dream synthesis child's check before any model call: refused (`session_source_changed: <reason>`, no retry) when its
* pinned file no longer resolves to the job's source or incarnation. A child queued before the pin existed is checked
- * from `dream_output.raw_source` and the configured corpus dir.
+ * from `dream_output.raw_source` and the configured corpus dir, which only a managed brain's children carry; an unmanaged
+ * child queued before the pin has neither and runs unchecked.
*/
export async function refuseChangedSessionSource(engine: BrainEngine,
data: { corpus_source?: CorpusSourcePin; dream_output?: { raw_source: string }; source_id?: string }): Promise<void> {
diff --git a/src/core/page-body/streamed-import.ts b/src/core/page-body/streamed-import.ts
index e41646c2a1..3cb2b32b21 100644
--- a/src/core/page-body/streamed-import.ts
+++ b/src/core/page-body/streamed-import.ts
@@ -345,8 +345,8 @@ async function importScanned(engine: BrainEngine, input: { file: FileHandle; bom
const parsed = { ...scan.parsed, compiled_truth: '', timeline: '' } as ParsedMarkdown;
const frontmatterHold = classifyImportHold(parsed);
if (frontmatterHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: frontmatterHold.message, refusal: frontmatterHold, skip_reason: 'invalid_frontmatter' };
- // #7848: a file import holds a private request page privacy would not read; a page write (`content`) refuses it at its own guard.
- const privateHold = opts.content ? null : noncanonicalPrivateHold('', parsed.frontmatter);
+ // #7848: a private request page privacy would not read is held; only an upload session (`content` plus `coordinated`) refuses it at its own guard.
+ const privateHold = opts.content && opts.coordinated ? null : noncanonicalPrivateHold('', parsed.frontmatter);
if (privateHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: privateHold.message, refusal: privateHold, skip_reason: privateHold.code };
let slug = expectedSlug;
if (!expectedSlug || parsed.slug !== expectedSlug) {
diff --git a/test/cli-page-writes.test.ts b/test/cli-page-writes.test.ts
index b44d4b47a3..1e2147caed 100644
--- a/test/cli-page-writes.test.ts
+++ b/test/cli-page-writes.test.ts
@@ -222,6 +222,14 @@ for (const backend of testBackends()) {
expect(await uploads()).toBe(before);
}, 600_000);
+ test('local put --file over the page ceiling on an unmanaged brain refuses a noncanonical private request, as a smaller put does, and writes nothing (#7848)', async () => {
+ await sql('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ const put = await cli(['put', 'notes/huge-private', '--file', file('huge-private.md', `---\nVisibility: Private\n---\n${hugeBody('Huge private')}`)]);
+ expect(put.code).not.toBe(0);
+ expect(put.stderr).toContain('visibility_not_applied');
+ expect(await sql<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE slug = $1', ['notes/huge-private'])).toEqual([{ n: 0 }]);
+ }, 600_000);
+
test('local capture --file over the page ceiling on an unmanaged brain streams in under the capture slug and frontmatter once the write gate allows it, with no upload session', async () => {
const before = await uploads();
const raw = hugeBody('Captured');
diff --git a/test/synthesize-source-routing-realpath-7871.test.ts b/test/synthesize-source-routing-realpath-7871.test.ts
index 4af2dc4b50..856a897cda 100644
--- a/test/synthesize-source-routing-realpath-7871.test.ts
+++ b/test/synthesize-source-routing-realpath-7871.test.ts
@@ -139,4 +139,15 @@ describe('#7871 corpus files are recognized through symlinks', () => {
expect(routing.held).toEqual({ corpus_path_unresolvable: 1 });
});
}, 60_000);
+
+ test('a path outside the corpus that no longer exists is not a corpus file, so it is not held as a captured session', async () => {
+ await brain(async (engine, dir) => {
+ const corpus = join(dir, 'corpus');
+ mkdirSync(corpus);
+ const gone = join(dir, 'meetings', 'gone.txt');
+ const { kept, routing } = await route(engine, [gone], corpus, 'default');
+ expect(kept).toEqual([gone]);
+ expect(routing.held).toEqual({});
+ });
+ }, 60_000);
});
diff --git a/test/visibility-noncanonical-surfaces-7848.test.ts b/test/visibility-noncanonical-surfaces-7848.test.ts
new file mode 100644
index 0000000000..44ff46548f
--- /dev/null
+++ b/test/visibility-noncanonical-surfaces-7848.test.ts
@@ -0,0 +1,95 @@
+/**
+ * Value: protects=file-ingestion surfaces name a noncanonical private request as visibility_noncanonical; fails_when=a surface drops holdNoncanonicalPrivate or its fix case.
+ * Value: why_new=visibility-noncanonical-hold-7848 drives sync, import and doctor, not validate --importable, the hold fix, the unindexed scan or the streamed import; seam=none; no_claim=sync (covered there).
+ *
+ * #7848 surfaces beside sync and import: `gbrain frontmatter validate --importable` (what a pre-commit hook sees), the hold's
+ * next step and fault class, the unindexed-files scan that finds committed files an older release never held, and the
+ * streamed (segmented-body) import, including a local page write that streams without an upload session.
+ * PGLite only; the sync and import paths live in test/visibility-noncanonical-hold-7848.test.ts.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { runFrontmatter } from '../src/commands/frontmatter.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { readPageBodyPolicy } from '../src/core/page-body/limits.ts';
+import { importStreamedMarkdownFile } from '../src/core/page-body/streamed-import.ts';
+import { claimWorktree } from '../src/core/persistence/ownership.ts';
+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
+import { gitHoldFix } from '../src/core/persistence/sync-holds.ts';
+import { classifySyncFault } from '../src/core/persistence/sync-fault-class.ts';
+import { findUnindexedFiles } from '../src/core/persistence/unindexed-files.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const note = (extra: string) => `---\ntitle: Note\ntype: note\n${extra}---\nA note about acme-example.\n`;
+const UPPER = note('Visibility: Private\n');
+const CANONICAL = note('visibility: private\n');
+
+async function validateImportable(content: string): Promise<{ hold?: { code?: string } | null }> {
+ const logOrig = console.log;
+ const prevExitCode = process.exitCode;
+ const out: string[] = [];
+ console.log = (...a: unknown[]) => { out.push(a.map(String).join(' ')); };
+ try { await runFrontmatter(['validate', '--stdin', '--importable', '--json'], { stdin: Readable.from([Buffer.from(content)]) }); } finally { console.log = logOrig; process.exitCode = prevExitCode ?? 0; }
+ const result = JSON.parse(out.join('\n')).results[0];
+ return { hold: result.hold };
+}
+
+describe('#7848 surfaces beside sync and import', () => {
+ let engine: PGLiteEngine;
+ const home = mkdtempSync(join(tmpdir(), 'gbrain-7848-surfaces-'));
+ beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); }, 120_000);
+ afterAll(async () => { await disposePersistenceConsumer(engine); await engine.disconnect(); rmSync(home, { recursive: true, force: true }); });
+
+ test('validate --importable names visibility_noncanonical for a noncanonical private request and passes a canonical one', async () => {
+ expect((await validateImportable(UPPER)).hold).toMatchObject({ code: 'visibility_noncanonical' });
+ expect((await validateImportable(CANONICAL)).hold ?? null).toBeNull();
+ });
+
+ test('the hold is a user-owned fix that names the lowercase line, and the fault table needs a human for it', () => {
+ const fix = gitHoldFix({ source_id: 'source-a', path: 'notes/upper.md', slug: 'notes/upper', code: 'visibility_noncanonical', meta: {} } as Parameters<typeof gitHoldFix>[0]);
+ expect(fix).toMatchObject({ actor: 'user', consent: [] });
+ expect(fix.user_message).toContain('visibility: private');
+ expect(fix.user_message).toContain('gbrain sync --source source-a --no-pull');
+ expect(classifySyncFault({ code: 'visibility_noncanonical' })).toMatchObject({ class: 'page', needs_human: true, safe_actions: ['none'] });
+ });
+
+ test('the unindexed-files scan screens a committed file that asks for private noncanonically as visibility_noncanonical, not clean', () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home }, async () => {
+ const repo = join(home, 'repo');
+ mkdirSync(join(repo, 'notes'), { recursive: true });
+ const git = (...args: string[]) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+ git('init', '-q');
+ writeFileSync(join(repo, 'notes/upper.md'), UPPER);
+ writeFileSync(join(repo, 'notes/canonical.md'), CANONICAL);
+ git('add', '-A'); git('-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'content');
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ await engine.executeRaw("INSERT INTO sources(id,name,local_path,config,last_commit) VALUES('source-a','source-a',$1,'{}',$2)", [repo, git('rev-parse', 'HEAD')]);
+ await claimWorktree(engine, 'source-a', repo);
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
+ const scan = await findUnindexedFiles(engine, 'source-a');
+ expect(scan.files.map(file => [file.path, file.code]).sort()).toEqual([['notes/canonical.md', 'clean'], ['notes/upper.md', 'visibility_noncanonical']]);
+ }), 120_000);
+
+ test('the streamed (segmented-body) import holds a noncanonical private request, a local page write streaming without an upload session included, and stores the fixed file private', async () => {
+ const path = join(home, 'upper-large.md');
+ const body = '\n' + 'A line about acme-example.\n'.repeat(2000);
+ writeFileSync(path, UPPER + body);
+ const policy = { ...(await readPageBodyPolicy(engine)), segmented: true };
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
+ try {
+ expect(await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true }, statSync(path), policy))
+ .toMatchObject({ status: 'skipped', skip_reason: 'visibility_noncanonical' });
+ expect(await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true, inferFrontmatter: false, content: {} }, statSync(path), policy))
+ .toMatchObject({ status: 'skipped', skip_reason: 'visibility_noncanonical' });
+ expect(await engine.getPage('notes/upper-large')).toBeNull();
+ writeFileSync(path, CANONICAL + body);
+ expect((await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true }, statSync(path), policy)).status).not.toBe('skipped');
+ expect((await engine.getPage('notes/upper-large'))?.frontmatter).toMatchObject({ visibility: 'private' });
+ } finally {
+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
+ }
+ }, 120_000);
+});