Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: pre-landing review fixes (#7848, #7850, #7871)
garrytan748bf63162fc
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4f5ae4d72b..a92102d199 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -40,10 +40,11 @@ gbrain dream --dry-run --phase synthesize # preview now records nothing- Sessions an earlier release already froze, and pages or atoms already written into the wrong source, are not changed or moved. Repair stays a separate, preview-first step.- Held sessions and held generation members are counted in the phase summary and `details.source_routing`, with the command that releases them.+- A synthesis job queued by an earlier release is checked again only on a brain with managed persistence, where the job records its transcript. On other brains, let the waiting dream jobs (`gbrain jobs list --status waiting`) finish before you add, archive or recreate sources.### Private files in another spelling-**Git sync and `gbrain import` hold a file that asks for `visibility: private` in a spelling privacy does not read, instead of publishing it world-readable (#7848).** Page privacy reads only the exact lowercase `visibility: private`. A file that wrote `Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key used to sync or import as a page anyone could read. Now classic and managed sync hold it as `visibility_noncanonical` (the rest of the source syncs, and an existing page keeps its last imported version), and `gbrain import`, with or without a Git checkout, refuses it and imports the rest. Write `visibility: private` on its own frontmatter line and commit, or import the file again. `gbrain doctor` (`git_held_files`) counts the held files and the stored pages that already ask for private this way; nothing rewrites them.+**Git sync and `gbrain import` hold a file that asks for `visibility: private` in a spelling privacy does not read, instead of publishing it world-readable (#7848).** Page privacy reads only the exact lowercase `visibility: private`. A file that wrote `Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key used to sync or import as a page anyone could read. Now classic and managed sync hold it as `visibility_noncanonical` (the rest of the source syncs, and an existing page keeps its last imported version), and `gbrain import`, with or without a Git checkout, refuses it and imports the rest. Write `visibility: private` on its own frontmatter line and commit, or import the file again. `gbrain doctor` (`git_held_files`) counts the held files and the stored pages that already ask for private this way; nothing rewrites them. On a brain without managed persistence, `gbrain put --file` and `gbrain capture --file` of a page over the single-string ceiling now refuse such a request with `visibility_not_applied`, as a smaller write does.### Itemized changesdiff --git a/src/cli/commands/append.ts b/src/cli/commands/append.tsindex f4986aab76..1317f06f38 100644--- a/src/cli/commands/append.ts+++ b/src/cli/commands/append.ts@@ -456,7 +456,7 @@ async function streamLocalPage(connect: () => Promise<BrainEngine>, path: stringassertPageRevision(await engine.readPageSnapshot(slug, { sourceId, includeDeleted: true }), engineMutationPrecondition(parseMutationPrecondition(params)));const result = await importStreamedMarkdownFile(engine, resolve(path), relativePath, { sourceId, inferFrontmatter: false, content }, stat, policy);if (result.status === 'skipped' && result.error) {- throw opError(result.refusal?.code === 'invalid_frontmatter' ? 'invalid_frontmatter' : 'invalid_params', result.error,+ throw opError(result.refusal?.code === 'invalid_frontmatter' ? 'invalid_frontmatter' : result.refusal?.code === 'visibility_noncanonical' ? 'visibility_not_applied' : 'invalid_params', result.error,`${path} was not written as ${slug}: fix what the message names (or split the file into smaller pages) and run the command again.`);}return result;diff --git a/src/commands/doctor/checks/git-holds.ts b/src/commands/doctor/checks/git-holds.tsindex db8f71ef9f..00085d4a59 100644--- a/src/commands/doctor/checks/git-holds.ts+++ b/src/commands/doctor/checks/git-holds.ts@@ -121,7 +121,7 @@ async function readNoncanonicalPrivate(engine: BrainEngine, sourceIds?: string[]const candidates = await engine.executeRaw<{ source_id: string; slug: string; frontmatter: Record<string, unknown> | string | null }>(`SELECT p.source_id, p.slug, p.frontmatterFROM pages p JOIN sources s ON s.id=p.source_idWHERE p.deleted_at IS NULL AND s.archived IS NOT TRUE AND ($1::text[] IS NULL OR p.source_id=ANY($1::text[])) AND jsonb_typeof(p.frontmatter)='object'- AND p.frontmatter::text ILIKE '%visibility%' AND p.frontmatter::text ILIKE '%private%' AND p.frontmatter->>'visibility' IS DISTINCT FROM 'private'+ AND EXISTS (SELECT 1 FROM jsonb_object_keys(p.frontmatter) k WHERE lower(k)='visibility') AND p.frontmatter->>'visibility' IS DISTINCT FROM 'private'ORDER BY p.source_id, p.slug`, [ids]);const stored = candidates.filter(row => noncanonicalPrivateRequest('', typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter));return { held: Number(heldRow?.n ?? 0), stored: stored.length, first: stored.slice(0, SAMPLE_PATHS).map(({ source_id, slug }) => ({ source_id, slug })) };diff --git a/src/core/behavior-change-notice.ts b/src/core/behavior-change-notice.tsindex 38d238253d..d616610f20 100644--- a/src/core/behavior-change-notice.ts+++ b/src/core/behavior-change-notice.ts@@ -1330,11 +1330,11 @@ export const BEHAVIOR_CHANGES: ReadonlyArray<{ since: string; text: ChangeText;{ since: '0.60.482.0', id: 'extract_atoms_session_source_7846', text: 'Dream `extract_atoms` writes a captured session\'s atoms only into the source its session recorded (the `.src-<id>` name in the corpus `sourced/` folder, or the session\'s frozen record), never into `default`. Each source\'s run now reads the session corpus and keeps only its own sessions (`gbrain dream --source <id> --phase extract_atoms`); meeting transcripts stay with `default`. A session whose source never resolved, and a file written before sessions recorded their source on a brain with more than one source (already-swept ones included), is held until `gbrain sweep --assign-corpus <source>` previews and `--apply` maps it. The phase summary and `details.source_routing` count both. Atoms an earlier release already wrote into the wrong source are not moved (#7846).' },// #7868, #7850, #7869, #7870, #7871 frozen-source privacy wave. `since` is stamped at the merge slot.{ since: 'TBD', id: 'corpus_session_tier4_only_7868', text: 'A captured session whose source was never recorded is no longer filed under `sources.default`, the only non-default source or `default` when the brain resolves it later: only a registered source whose folder contains the session\'s working directory counts. Any other session stays held as `corpus_source_unresolved` until `gbrain sweep --assign-corpus <source>` previews and `--apply` maps it. A brain with one source still uses that source, without recording it, so a source added later holds the session instead. Sessions an earlier release already recorded keep their source (#7868).' },- { since: 'TBD', id: 'synthesize_session_recheck_7850', text: 'A dream synthesis job checks its session\'s source again when it runs and is refused (`session_source_changed`) before any model call when the session now belongs to another source or its source was deleted and recreated; jobs queued by an earlier release are checked too. `gbrain dream --dry-run` no longer records a session\'s source while previewing synthesize or extract_atoms (#7850).' },+ { since: 'TBD', id: 'synthesize_session_recheck_7850', text: 'A dream synthesis job checks its session\'s source again when it runs and is refused (`session_source_changed`) before any model call when the session now belongs to another source or its source was deleted and recreated; on a managed brain, jobs queued by an earlier release are checked too. `gbrain dream --dry-run` no longer records a session\'s source while previewing synthesize or extract_atoms (#7850).' },{ since: 'TBD', id: 'atom_retry_resume_session_source_7869_7870', text: 'An explicit atom retry (`gbrain jobs submit extract-atoms-drain` with `retryRequestId`) of a captured session that does not belong to the retry\'s source is refused with `transcript_source_mismatch` and starts nothing. A crashed atom generation whose pending members came from such a session is not resumed: the members are held, their atom pages left as they are, and counted in `details.source_routing.generation_members_held` (#7869, #7870).' },{ since: 'TBD', id: 'corpus_realpath_7871', text: 'Dream synthesis and extract_atoms recognize a session-corpus file through symlinks (a meeting folder or corpus path that points into the corpus, a linked `sourced/` folder), so such a file is routed to its session\'s source instead of being read as an ordinary transcript; a listed file whose real path cannot be read is held as `corpus_path_unresolvable` (#7871).' },// #7848 non-canonical private requests on file ingestion. `since` is stamped at the merge slot.- { since: 'TBD', id: 'visibility_noncanonical_hold_7848', text: 'Git sync (classic and managed) and `gbrain import` (with or without a Git checkout) hold a file whose frontmatter asks for `visibility: private` in a spelling page privacy does not read (`Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key) as `visibility_noncanonical` instead of storing a world-readable page; the rest of the source syncs and an existing page keeps its last imported version. Write `visibility: private` on its own frontmatter line (lowercase) and commit, or import the file again. Doctor `git_held_files` counts the held files and the stored pages that already ask for private this way (`details.visibility_noncanonical`); nothing rewrites them.' },+ { since: 'TBD', id: 'visibility_noncanonical_hold_7848', text: 'Git sync (classic and managed) and `gbrain import` (with or without a Git checkout) hold a file whose frontmatter asks for `visibility: private` in a spelling page privacy does not read (`Visibility: private`, `visibility: Private`, a block scalar, a list, a flow mapping or a merge key) as `visibility_noncanonical` instead of storing a world-readable page; the rest of the source syncs and an existing page keeps its last imported version. Write `visibility: private` on its own frontmatter line (lowercase) and commit, or import the file again. Doctor `git_held_files` counts the held files and the stored pages that already ask for private this way (`details.visibility_noncanonical`); nothing rewrites them. On a brain without managed persistence, `gbrain put --file` and `gbrain capture --file` of a page over the single-string ceiling refuse such a request with `visibility_not_applied`, as a smaller write does.' },];/** The newest disclosed change's release: the notice id moves only when a release adds rows. */diff --git a/src/core/cycle/synthesize-source-routing.ts b/src/core/cycle/synthesize-source-routing.tsindex 7f675b0252..1ce5f6e42b 100644--- a/src/core/cycle/synthesize-source-routing.ts+++ b/src/core/cycle/synthesize-source-routing.ts@@ -26,7 +26,7 @@* (`transcriptSourceHold`, #7869, #7870). Dry runs resolve read-only.*/-import { realpathSync } from 'node:fs';+import { lstatSync, realpathSync } from 'node:fs';import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path';import type { BrainEngine } from '../engine.ts';import type { PhaseResult } from '../cycle.ts';@@ -66,6 +66,7 @@ function corpusLocation(filePath: string, corpusDir: string): CorpusFileLocationif (realRoot === null) return lexical;const realSpool = realpathOrMissing(corpusSpoolDir(root));const realFile = realpathOrMissing(filePath);+ if (realFile === null && !lstatSync(filePath, { throwIfNoEntry: false })) return lexical;if (realRoot === undefined || realSpool === undefined || !realFile) return 'unresolvable';const real = locate(realFile, realRoot, realSpool);return lexical === 'spool' || real === 'spool' ? 'spool' : lexical ?? real;@@ -119,7 +120,8 @@ export async function corpusSourceChanged(engine: BrainEngine, pin: { raw_source/*** A dream synthesis child's check before any model call: refused (`session_source_changed: <reason>`, no retry) when its* pinned file no longer resolves to the job's source or incarnation. A child queued before the pin existed is checked- * from `dream_output.raw_source` and the configured corpus dir.+ * from `dream_output.raw_source` and the configured corpus dir, which only a managed brain's children carry; an unmanaged+ * child queued before the pin has neither and runs unchecked.*/export async function refuseChangedSessionSource(engine: BrainEngine,data: { corpus_source?: CorpusSourcePin; dream_output?: { raw_source: string }; source_id?: string }): Promise<void> {diff --git a/src/core/page-body/streamed-import.ts b/src/core/page-body/streamed-import.tsindex e41646c2a1..3cb2b32b21 100644--- a/src/core/page-body/streamed-import.ts+++ b/src/core/page-body/streamed-import.ts@@ -345,8 +345,8 @@ async function importScanned(engine: BrainEngine, input: { file: FileHandle; bomconst parsed = { ...scan.parsed, compiled_truth: '', timeline: '' } as ParsedMarkdown;const frontmatterHold = classifyImportHold(parsed);if (frontmatterHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: frontmatterHold.message, refusal: frontmatterHold, skip_reason: 'invalid_frontmatter' };- // #7848: a file import holds a private request page privacy would not read; a page write (`content`) refuses it at its own guard.- const privateHold = opts.content ? null : noncanonicalPrivateHold('', parsed.frontmatter);+ // #7848: a private request page privacy would not read is held; only an upload session (`content` plus `coordinated`) refuses it at its own guard.+ const privateHold = opts.content && opts.coordinated ? null : noncanonicalPrivateHold('', parsed.frontmatter);if (privateHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: privateHold.message, refusal: privateHold, skip_reason: privateHold.code };let slug = expectedSlug;if (!expectedSlug || parsed.slug !== expectedSlug) {diff --git a/test/cli-page-writes.test.ts b/test/cli-page-writes.test.tsindex b44d4b47a3..1e2147caed 100644--- a/test/cli-page-writes.test.ts+++ b/test/cli-page-writes.test.ts@@ -222,6 +222,14 @@ for (const backend of testBackends()) {expect(await uploads()).toBe(before);}, 600_000);+ test('local put --file over the page ceiling on an unmanaged brain refuses a noncanonical private request, as a smaller put does, and writes nothing (#7848)', async () => {+ await sql('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ const put = await cli(['put', 'notes/huge-private', '--file', file('huge-private.md', `---\nVisibility: Private\n---\n${hugeBody('Huge private')}`)]);+ expect(put.code).not.toBe(0);+ expect(put.stderr).toContain('visibility_not_applied');+ expect(await sql<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE slug = $1', ['notes/huge-private'])).toEqual([{ n: 0 }]);+ }, 600_000);+test('local capture --file over the page ceiling on an unmanaged brain streams in under the capture slug and frontmatter once the write gate allows it, with no upload session', async () => {const before = await uploads();const raw = hugeBody('Captured');diff --git a/test/synthesize-source-routing-realpath-7871.test.ts b/test/synthesize-source-routing-realpath-7871.test.tsindex 4af2dc4b50..856a897cda 100644--- a/test/synthesize-source-routing-realpath-7871.test.ts+++ b/test/synthesize-source-routing-realpath-7871.test.ts@@ -139,4 +139,15 @@ describe('#7871 corpus files are recognized through symlinks', () => {expect(routing.held).toEqual({ corpus_path_unresolvable: 1 });});}, 60_000);++ test('a path outside the corpus that no longer exists is not a corpus file, so it is not held as a captured session', async () => {+ await brain(async (engine, dir) => {+ const corpus = join(dir, 'corpus');+ mkdirSync(corpus);+ const gone = join(dir, 'meetings', 'gone.txt');+ const { kept, routing } = await route(engine, [gone], corpus, 'default');+ expect(kept).toEqual([gone]);+ expect(routing.held).toEqual({});+ });+ }, 60_000);});diff --git a/test/visibility-noncanonical-surfaces-7848.test.ts b/test/visibility-noncanonical-surfaces-7848.test.tsnew file mode 100644index 0000000000..44ff46548f--- /dev/null+++ b/test/visibility-noncanonical-surfaces-7848.test.ts@@ -0,0 +1,95 @@+/**+ * Value: protects=file-ingestion surfaces name a noncanonical private request as visibility_noncanonical; fails_when=a surface drops holdNoncanonicalPrivate or its fix case.+ * Value: why_new=visibility-noncanonical-hold-7848 drives sync, import and doctor, not validate --importable, the hold fix, the unindexed scan or the streamed import; seam=none; no_claim=sync (covered there).+ *+ * #7848 surfaces beside sync and import: `gbrain frontmatter validate --importable` (what a pre-commit hook sees), the hold's+ * next step and fault class, the unindexed-files scan that finds committed files an older release never held, and the+ * streamed (segmented-body) import, including a local page write that streams without an upload session.+ * PGLite only; the sync and import paths live in test/visibility-noncanonical-hold-7848.test.ts.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { execFileSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Readable } from 'node:stream';+import { runFrontmatter } from '../src/commands/frontmatter.ts';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { readPageBodyPolicy } from '../src/core/page-body/limits.ts';+import { importStreamedMarkdownFile } from '../src/core/page-body/streamed-import.ts';+import { claimWorktree } from '../src/core/persistence/ownership.ts';+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';+import { gitHoldFix } from '../src/core/persistence/sync-holds.ts';+import { classifySyncFault } from '../src/core/persistence/sync-fault-class.ts';+import { findUnindexedFiles } from '../src/core/persistence/unindexed-files.ts';+import { withEnv } from './helpers/with-env.ts';++const note = (extra: string) => `---\ntitle: Note\ntype: note\n${extra}---\nA note about acme-example.\n`;+const UPPER = note('Visibility: Private\n');+const CANONICAL = note('visibility: private\n');++async function validateImportable(content: string): Promise<{ hold?: { code?: string } | null }> {+ const logOrig = console.log;+ const prevExitCode = process.exitCode;+ const out: string[] = [];+ console.log = (...a: unknown[]) => { out.push(a.map(String).join(' ')); };+ try { await runFrontmatter(['validate', '--stdin', '--importable', '--json'], { stdin: Readable.from([Buffer.from(content)]) }); } finally { console.log = logOrig; process.exitCode = prevExitCode ?? 0; }+ const result = JSON.parse(out.join('\n')).results[0];+ return { hold: result.hold };+}++describe('#7848 surfaces beside sync and import', () => {+ let engine: PGLiteEngine;+ const home = mkdtempSync(join(tmpdir(), 'gbrain-7848-surfaces-'));+ beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); }, 120_000);+ afterAll(async () => { await disposePersistenceConsumer(engine); await engine.disconnect(); rmSync(home, { recursive: true, force: true }); });++ test('validate --importable names visibility_noncanonical for a noncanonical private request and passes a canonical one', async () => {+ expect((await validateImportable(UPPER)).hold).toMatchObject({ code: 'visibility_noncanonical' });+ expect((await validateImportable(CANONICAL)).hold ?? null).toBeNull();+ });++ test('the hold is a user-owned fix that names the lowercase line, and the fault table needs a human for it', () => {+ const fix = gitHoldFix({ source_id: 'source-a', path: 'notes/upper.md', slug: 'notes/upper', code: 'visibility_noncanonical', meta: {} } as Parameters<typeof gitHoldFix>[0]);+ expect(fix).toMatchObject({ actor: 'user', consent: [] });+ expect(fix.user_message).toContain('visibility: private');+ expect(fix.user_message).toContain('gbrain sync --source source-a --no-pull');+ expect(classifySyncFault({ code: 'visibility_noncanonical' })).toMatchObject({ class: 'page', needs_human: true, safe_actions: ['none'] });+ });++ test('the unindexed-files scan screens a committed file that asks for private noncanonically as visibility_noncanonical, not clean', () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home }, async () => {+ const repo = join(home, 'repo');+ mkdirSync(join(repo, 'notes'), { recursive: true });+ const git = (...args: string[]) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();+ git('init', '-q');+ writeFileSync(join(repo, 'notes/upper.md'), UPPER);+ writeFileSync(join(repo, 'notes/canonical.md'), CANONICAL);+ git('add', '-A'); git('-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'content');+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ await engine.executeRaw("INSERT INTO sources(id,name,local_path,config,last_commit) VALUES('source-a','source-a',$1,'{}',$2)", [repo, git('rev-parse', 'HEAD')]);+ await claimWorktree(engine, 'source-a', repo);+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');+ const scan = await findUnindexedFiles(engine, 'source-a');+ expect(scan.files.map(file => [file.path, file.code]).sort()).toEqual([['notes/canonical.md', 'clean'], ['notes/upper.md', 'visibility_noncanonical']]);+ }), 120_000);++ test('the streamed (segmented-body) import holds a noncanonical private request, a local page write streaming without an upload session included, and stores the fixed file private', async () => {+ const path = join(home, 'upper-large.md');+ const body = '\n' + 'A line about acme-example.\n'.repeat(2000);+ writeFileSync(path, UPPER + body);+ const policy = { ...(await readPageBodyPolicy(engine)), segmented: true };+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ try {+ expect(await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true }, statSync(path), policy))+ .toMatchObject({ status: 'skipped', skip_reason: 'visibility_noncanonical' });+ expect(await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true, inferFrontmatter: false, content: {} }, statSync(path), policy))+ .toMatchObject({ status: 'skipped', skip_reason: 'visibility_noncanonical' });+ expect(await engine.getPage('notes/upper-large')).toBeNull();+ writeFileSync(path, CANONICAL + body);+ expect((await importStreamedMarkdownFile(engine, path, 'notes/upper-large.md', { noEmbed: true }, statSync(path), policy)).status).not.toBe('skipped');+ expect((await engine.getPage('notes/upper-large'))?.frontmatter).toMatchObject({ visibility: 'private' });+ } finally {+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');+ }+ }, 120_000);+});