Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Opt-in markdown write-through for pages the dream cycle generates on an
* unmanaged brain (#5041): atoms (`extract_atoms`) and concepts
* (`synthesize_concepts`). Off by default, so the cycle never adds files to a
* user's repository unasked; managed brains publish these pages through
* maintenance publication instead and never reach this module.
*
* Enabled by `cycle.extract_atoms.write_through` /
* `cycle.synthesize_concepts.write_through`. Each written page goes through
* `writePageThrough` (honouring `sync.write_through=off`, source containment
* and case collisions). A page whose target is ignored by git or sits in a
* declared `storage.db_only` directory (which sync gitignores) stays
* database-only. The first run after the key is enabled also writes the
* generated pages that have no file yet, a bounded batch per run. A page the
* phase retires has its file removed, so sync cannot bring it back.
*
* #8073: in a Git checkout those files are committed, one commit per batch
* through the shared-checkout door (`commitPathsLocked`), and a refused commit
* rolls the batch's files back (`commitDerivedChanges`), so the cycle never
* leaves untracked or modified files that make the scheduled sync skip.
*
* #6982: a file is replaced or removed only when it holds nothing the
* database lacks (`replaceableFile`: absent, or equal to the page or one of
* its recent versions), and the write is bound to the bytes compared. A file
* holding a hand edit or an unindexed file at the path is left as it is and
* the page is reported in the hook's `held` list (`file_database_drift`).
*/
import { execFileSync } from '../spawn.ts';
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join, relative } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { PhaseResult } from '../cycle.ts';
import type { Page } from '../types.ts';
import { throwIfAborted } from '../abort-check.ts';
import { validateSourceId } from '../utils.ts';
import { isDbOnly, loadStorageConfig } from '../storage-config.ts';
import { deletePageThrough, isWriteThroughDisabled, resolvePageWriteTarget, writePageThrough, type PageWriteTarget } from '../write-through.ts';
import { readOnlyCheckout } from '../persistence/mirror-read-only.ts';
import { gitChildEnv, literalPathspecs } from '../git-env.ts';
import { classifyGitCheckout, type GitCheckoutClass } from '../git-checkout.ts';
import { acquireCommitLock, acquireCommitLockAsync, checkoutRoot, commitAndPush, commitPathsLocked, COMMIT_LOCK_ENV, notePending, scopePaths, settlePending } from '../git/commit-push.ts';
import { managedFilesystemRootFor, refreshManagedFilesystemRoots } from '../persistence/filesystem-guard.ts';
import { withSourceFilesystemLock } from '../minions/source-filesystem.ts';
import { fileDatabaseDriftToken, replaceableFile } from '../facts/file-body-guard.ts';
export const DERIVED_WRITE_THROUGH_KEYS = {
extract_atoms: 'cycle.extract_atoms.write_through',
synthesize_concepts: 'cycle.synthesize_concepts.write_through',
} as const;
/** Pages without a file written per run by the enable-time backfill. */
export const DERIVED_BACKFILL_LIMIT = 200;
const ON_VALUES = new Set(['true', '1', 'yes', 'on']);
export async function derivedWriteThroughEnabled(engine: BrainEngine, phase: keyof typeof DERIVED_WRITE_THROUGH_KEYS): Promise<boolean> {
const raw = await engine.getConfig(DERIVED_WRITE_THROUGH_KEYS[phase]).catch(() => null);
return raw != null && ON_VALUES.has(raw.trim().toLowerCase());
}
function gitIgnores(root: string, filePath: string): boolean {
try {
execFileSync('git', ['-C', root, 'check-ignore', '-q', '--', relative(root, filePath)], { env: gitChildEnv(), stdio: 'ignore' });
return true;
} catch {
return false;
}
}
/**
* The checkout file a generated page publishes to, or why it has none this
* module writes: write-through off (`disabled_by_config`), no resolvable
* target, or a target ignored by git or inside a declared `storage.db_only`
* directory (`not_canonical`). `includeDeleted` resolves a retired page's
* recorded path.
*/
export async function derivedFileTarget(engine: BrainEngine, slug: string, sourceId: string,
opts: { includeDeleted?: boolean } = {}): Promise<{ ok: true; target: Extract<PageWriteTarget, { ok: true }> } | { ok: false; skipped: string }> {
if (await isWriteThroughDisabled(engine)) return { ok: false, skipped: 'disabled_by_config' };
const target = await resolvePageWriteTarget(engine, slug, sourceId, opts);
if (!target.ok) return { ok: false, skipped: target.skipped };
let storage = null;
try { storage = loadStorageConfig(target.writeRoot); } catch { /* unreadable gbrain.yml: sync reports it */ }
if ((storage && isDbOnly(slug, storage)) || gitIgnores(target.writeRoot, target.filePath)) return { ok: false, skipped: 'not_canonical' };
return { ok: true, target };
}
/** One file a derived write changed (#8073): its bytes before and after (null: absent) and the source_path the write bound. */
interface DerivedFileChange { slug: string; sourceId: string; filePath: string; writeRoot: string; prior: string | null; after: string | null; bound: string | null }
/** A batch commit the checkout refused: the door's code and message, the batch's files (root-relative) and those left as they were. */
export interface DerivedCommitFailure { code: string; message: string; root: string; paths: string[]; kept: string[] }
/**
* Test seams. `beforeCommit` runs after a batch's files are written and before its commit; a throw stands in for a crash
* (no rollback). `checkoutClass` replaces `classifyGitCheckout` for a write root (an `unknown` answer needs a failed lookup).
*/
export const derivedWriteSeams: { beforeCommit?: (root: string) => void; checkoutClass?: (root: string) => GitCheckoutClass } = {};
/** Whether `root` sits in a Git checkout this module commits into (`classifyGitCheckout`, or the test seam). */
function checkoutClassOf(root: string): GitCheckoutClass {
return (derivedWriteSeams.checkoutClass ?? classifyGitCheckout)(root);
}
/** How long a batch waits, without blocking the event loop, for the checkout's commit lock before it rolls back. */
const DERIVED_COMMIT_LOCK_WAIT_MS = 30_000;
/** #8092: the synchronous take once the lock was seen free, and the rollback's wait for it (as the classic write-through's). */
const DERIVED_COMMIT_LOCK_TAKE_MS = 2_000;
/** Whether a parent git commit-push holds this checkout's commit lock and exported its marker (COMMIT_LOCK_ENV) to this process. */
const commitLockHeld = (): boolean => !!process.env[COMMIT_LOCK_ENV];
/** #8092: op_checkpoints op naming a generated page whose file change a refused commit rolled back or left unwritten; the phase's next hook retries it. */
const RETRY_OP = 'derived-write-retry';
const retryKey = (sourceId: string, slug: string) => `${sourceId}:${slug}`;
async function markRetry(engine: BrainEngine, pages: ReadonlyArray<{ sourceId: string; slug: string }>): Promise<void> {
for (const { sourceId, slug } of pages) {
await engine.executeRaw(`INSERT INTO op_checkpoints(op, fingerprint, completed_keys) VALUES ($1, $2, $3::text::jsonb)
ON CONFLICT (op, fingerprint) DO UPDATE SET updated_at = now()`, [RETRY_OP, retryKey(sourceId, slug), JSON.stringify([{ source_id: sourceId, slug }])]);
}
}
async function clearRetry(engine: BrainEngine, sourceId: string, slugs: readonly string[]): Promise<void> {
if (slugs.length) await engine.executeRaw('DELETE FROM op_checkpoints WHERE op = $1 AND fingerprint = ANY($2::text[])', [RETRY_OP, slugs.map(slug => retryKey(sourceId, slug))]);
}
/** The changes of `changes` a failure lists (rolled back, or left for the pending ledger). */
const failedChanges = (changes: readonly DerivedFileChange[], failures: readonly DerivedCommitFailure[]) =>
changes.filter(c => failures.some(f => f.root === c.writeRoot && f.paths.includes(relative(c.writeRoot, c.filePath))));
/** The file write of `writeDerivedPageThrough`, uncommitted, with the change it made. */
async function writeDerivedFile(engine: BrainEngine, slug: string, sourceId: string,
opts: { expectedFileBytes?: string | null }): Promise<{ written: boolean; skipped?: string; error?: string; change?: DerivedFileChange }> {
const resolved = await derivedFileTarget(engine, slug, sourceId);
if (!resolved.ok) return { written: false, skipped: resolved.skipped };
let expectedFileBytes = opts.expectedFileBytes;
if (expectedFileBytes === undefined) {
const file = await replaceableFile(engine, sourceId, slug, resolved.target.filePath);
if (!file.ok) return { written: false, skipped: 'file_database_drift' };
expectedFileBytes = file.expectedFileBytes;
}
const [row] = await engine.executeRaw<{ source_path: string | null }>(
'SELECT source_path FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL', [sourceId, slug]);
// inlineOnly: a page stored in segments has its file streamed without a string the batch rollback could compare (`after`).
const result = await writePageThrough(engine, slug, { sourceId, expectedFileBytes, deferCommit: true, inlineOnly: true });
if (result.error) console.error(`[write-through] ${slug}: ${result.error}`);
if (!result.written || !result.path) return result;
noteWritten(resolved.target.writeRoot, result.path);
return { ...result, change: { slug, sourceId, filePath: result.path, writeRoot: resolved.target.writeRoot, prior: expectedFileBytes,
after: result.content!, bound: row?.source_path === null ? resolved.target.sourcePathToBind : null } };
}
/**
* Record a file this module just wrote or removed in the checkout's pending write-through ledger (`notePending`), so a
* crash before its batch commits leaves it for the next locked commit there (`commitPathsLocked` sweeps the ledger's
* paths that are inside the source root and still differ from HEAD). Only paths gbrain wrote are ever recorded.
*/
function noteWritten(root: string, filePath: string): void {
try {
if (checkoutClassOf(root) !== 'git') return;
const top = checkoutRoot(root);
notePending(top, scopePaths(top, [filePath], top));
} catch { /* best-effort: the batch commit right after is the primary path */ }
}
/**
* Write one generated page's markdown file and commit it (#8073). Never
* throws; the result says whether a file was written, why not, or the write
* error. Ignored and db_only targets are skipped on purpose
* (`skipped: 'not_canonical'`). `expectedFileBytes` binds the write to the
* file the caller read (and already compared); without it the file must pass
* `replaceableFile` first, else nothing is written
* (`skipped: 'file_database_drift'`). A commit the checkout refuses rolls the
* file back and comes back as `error` (`<code>: <message>`).
*/
export async function writeDerivedPageThrough(engine: BrainEngine, slug: string, sourceId: string,
opts: { expectedFileBytes?: string | null } = {}): Promise<{ written: boolean; skipped?: string; error?: string }> {
const { change, ...result } = await writeDerivedFile(engine, slug, sourceId, opts);
if (!change) return result;
const [failure] = await commitDerivedChanges(engine, 'synthesize_concepts', [change]);
if (!failure) {
await clearRetry(engine, sourceId, [slug]);
return result;
}
await markRetry(engine, [change]);
return { written: false, error: `${failure.code}: ${failure.message}` };
}
/**
* Remove a retired generated page's file, uncommitted, when the retired page
* (or one of its versions) holds everything in it; the unlink is bound to the
* bytes compared. Returns `file_database_drift` when the file holds an edit
* the database lacks.
*/
async function deleteDerivedFile(engine: BrainEngine, slug: string, sourceId: string): Promise<{ removed: boolean; skipped?: string; change?: DerivedFileChange }> {
const resolved = await derivedFileTarget(engine, slug, sourceId, { includeDeleted: true });
if (!resolved.ok) return { removed: false, skipped: resolved.skipped };
const file = await replaceableFile(engine, sourceId, slug, resolved.target.filePath);
if (!file.ok) return { removed: false, skipped: 'file_database_drift' };
if (file.expectedFileBytes === null) return { removed: false, skipped: 'file_not_present' };
const result = await deletePageThrough(engine, slug, { sourceId, target: resolved.target, expectedFileBytes: file.expectedFileBytes, deferCommit: true });
if (!result.removed) return result;
noteWritten(resolved.target.writeRoot, resolved.target.filePath);
return { ...result, change: { slug, sourceId, filePath: resolved.target.filePath, writeRoot: resolved.target.writeRoot, prior: file.expectedFileBytes, after: null, bound: null } };
}
/**
* #8073: commit a batch of derived file changes through the shared-checkout commit door (`commitPathsLocked`: the commit
* lock, `commit --only` of exactly the named paths), one commit per checkout, under the source filesystem lock. Only paths
* that exist or are tracked are named (an untracked file the batch removed has nothing to commit). A directory that is not
* a Git checkout, or whose checkout lookup fails (`unknown`), has nothing to commit: its files are the record. A refused
* commit rolls the batch back (`rollBackDerived`) and comes back as a typed failure unless every file was committed meanwhile.
*/
async function commitDerivedChanges(engine: BrainEngine, label: string, changes: readonly DerivedFileChange[]): Promise<DerivedCommitFailure[]> {
const failures: DerivedCommitFailure[] = [];
for (const root of new Set(changes.map(c => c.writeRoot))) {
const batch = changes.filter(c => c.writeRoot === root);
if (checkoutClassOf(root) !== 'git') continue;
derivedWriteSeams.beforeCommit?.(root);
try {
await refreshManagedFilesystemRoots(engine);
const managed = managedFilesystemRootFor(root) !== null;
const commit = async () => {
const rels = batch.map(c => relative(root, c.filePath));
const tracked = new Set(gitPaths(root, ['ls-files', '-z', '--', ...literalPathspecs(rels)]));
const ignored = new Set(gitPaths(root, ['check-ignore', '-z', '--stdin', '--no-index'], rels.join('\0')));
const paths = batch.filter((c, i) => !ignored.has(rels[i]!) && (existsSync(c.filePath) || tracked.has(rels[i]!))).map(c => c.filePath);
if (!paths.length) return;
const message = `gbrain: ${label} write-through (${paths.length} generated file(s))`;
if (!commitLockHeld()) await (await acquireCommitLockAsync(checkoutRoot(root), DERIVED_COMMIT_LOCK_WAIT_MS)).release();
if (managed) commitAndPush({ repo: root, cwd: root, paths, message, lockWaitMs: DERIVED_COMMIT_LOCK_TAKE_MS });
else commitPathsLocked(root, paths, message, { lockWaitMs: DERIVED_COMMIT_LOCK_TAKE_MS });
};
if (managed) await commit();
else await withSourceFilesystemLock(engine, root, commit);
} catch (error) {
const failure = await rollBackDerived(engine, root, batch, error);
if (failure) failures.push(failure);
}
}
return failures;
}
/** NUL-separated paths a git listing command prints from `root` (exit 1, as `check-ignore` gives for none, is an empty list). */
function gitPaths(root: string, args: string[], input?: string): string[] {
try {
return execFileSync('git', ['-C', root, ...args], { env: gitChildEnv(), encoding: 'utf8', ...(input === undefined ? {} : { input }) }).split('\0').filter(Boolean);
} catch (error) {
if ((error as { status?: number }).status === 1) return [];
throw error;
}
}
/**
* Undo a batch whose commit was refused: each file still holding the bytes the batch wrote gets its prior bytes back (or is
* removed when it had none), and the source_path the write bound is cleared so the next run's backfill writes the page
* again. A file changed since (another writer's edit) is left as it is and listed in `kept`. A file whose change HEAD
* already holds (another writer's locked commit swept it in from the pending ledger) is committed, not rolled back:
* restoring it would leave an uncommitted revert of HEAD. Null when every file of the batch is committed.
* #8092: it runs under the checkout's commit lock (a short wait), so no writer commits between the HEAD check and the
* restore, and it settles the restored paths out of the pending ledger, so a later edit at one of them is never swept into
* a gbrain commit. When the lock stays busy nothing is restored: every file stays in the ledger for that writer's commit.
* Under a commit-push hook or child (BRAIN_COMMIT_LOCK_HELD naming this checkout's held lock) it runs under that lock.
*/
async function rollBackDerived(engine: BrainEngine, root: string, batch: readonly DerivedFileChange[], error: unknown): Promise<DerivedCommitFailure | null> {
const code = typeof (error as { code?: unknown }).code === 'string' ? (error as { code: string }).code : 'git_unavailable';
const message = error instanceof Error ? error.message : String(error);
const top = checkoutRoot(root);
const held = commitLockHeld() ? (() => { try { return acquireCommitLock(top, 0); } catch { return null; } })() : null;
const lock = held ? { release: async () => held.release() } : await acquireCommitLockAsync(top, DERIVED_COMMIT_LOCK_TAKE_MS).catch(() => null);
try {
const pending = batch.filter(change => !headHolds(root, change));
if (!pending.length) return null;
const paths = pending.map(c => relative(root, c.filePath));
if (!lock) {
console.error(`[write-through] commit refused (${code}): the commit lock stayed busy, so ${pending.length} generated file(s) are left for the next locked commit`);
return { code, message, root, paths, kept: paths };
}
const kept = await restoreDerived(engine, root, pending);
settlePending(top);
console.error(`[write-through] commit refused (${code}): ${pending.length - kept.length} generated file(s) rolled back, ${kept.length} left as they are`);
return { code, message, root, paths, kept };
} finally { await lock?.release(); }
}
/** Puts back the prior bytes of each change still holding the bytes it wrote and clears the source_path it bound; returns the root-relative paths left as they are. */
async function restoreDerived(engine: BrainEngine, root: string, pending: readonly DerivedFileChange[]): Promise<string[]> {
const kept: string[] = [];
for (const change of pending) {
try {
if ((existsSync(change.filePath) ? readFileSync(change.filePath, 'utf-8') : null) !== change.after) throw new Error('file_changed');
if (change.prior === null) rmSync(change.filePath, { force: true });
else {
mkdirSync(dirname(change.filePath), { recursive: true });
writeFileSync(`${change.filePath}.${process.pid}.tmp`, change.prior, 'utf-8');
renameSync(`${change.filePath}.${process.pid}.tmp`, change.filePath);
}
if (change.bound) {
await engine.executeRaw('UPDATE pages SET source_path = NULL WHERE source_id = $1 AND slug = $2 AND source_path = $3 AND deleted_at IS NULL',
[change.sourceId, change.slug, change.bound]);
}
} catch {
kept.push(relative(root, change.filePath));
}
}
return kept;
}
/** Whether HEAD of `root`'s checkout already holds the change's resulting bytes (absent for a removal); false when HEAD is unreadable. */
function headHolds(root: string, change: DerivedFileChange): boolean {
const run = (args: string[]) => execFileSync('git', ['-C', root, ...args], { env: gitChildEnv(), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
try { run(['rev-parse', '--verify', '-q', 'HEAD']); } catch { return false; }
try {
return run(['show', `HEAD:./${relative(root, change.filePath).split('\\').join('/')}`]) === change.after;
} catch {
return change.after === null;
}
}
/** `result` with its held page files reported: added to `details.file_holds`, a `warn` status unless it failed, and a summary note. */
export function reportFileHolds(result: PhaseResult, held: readonly string[] | undefined, noun: string): PhaseResult {
if (!held?.length) return result;
return { ...result, status: result.status === 'fail' ? 'fail' : 'warn',
details: { ...result.details, file_holds: [...new Set([...((result.details.file_holds as string[] | undefined) ?? []), ...held])] },
summary: `${result.summary} (${held.length} ${noun} file(s) held: ${[...new Set(held.map(h => /^\S+ \((\w+)/.exec(h)?.[1] ?? 'file_database_drift'))].join(', ')})` };
}
/** `result` with the hook's held files (`reportFileHolds`) and refused commits reported: `details.write_through_failed` and a `warn` status unless it failed. */
export function reportDerivedWriteThrough(result: PhaseResult, files: DerivedWriteThrough | null, noun: string): PhaseResult {
const reported = reportFileHolds(result, files?.held, noun);
if (!files?.failed.length) return reported;
const kept = files.failed.reduce((n, f) => n + f.kept.length, 0);
const count = files.failed.reduce((n, f) => n + f.paths.length, 0) - kept;
return { ...reported, status: reported.status === 'fail' ? 'fail' : 'warn',
details: { ...reported.details, write_through_failed: files.failed },
summary: `${reported.summary} (${noun} file commit refused: ${files.failed[0]!.code}; ${count} file(s) rolled back${kept ? `, ${kept} left as they are` : ''}; the next run writes them again)` };
}
/**
* The phase's write-through hook: `write` publishes pages and removes retired ones' files, one commit per call; `held`
* reports each page left alone, `failed` each refused commit (after which the hook writes nothing more this run). #8092:
* a page whose change was rolled back, or not written after a refusal, is marked for retry (`RETRY_OP`) and the next run's
* hook writes or removes its file again, an update of an existing file included.
*/
export interface DerivedWriteThrough {
write(slugs: readonly string[], retired?: readonly string[]): Promise<void>;
held: string[];
failed: DerivedCommitFailure[];
}
/**
* The phase's write-through hook, or null when it does not apply (key off,
* managed brain, dry run): writes the given pages and removes the files of
* the retired ones, then commits the batch. Creating it writes the bounded
* backfill of generated pages of the phase's type in the source that have no
* file yet.
*/
export async function derivedWriteThrough(
engine: BrainEngine,
phase: keyof typeof DERIVED_WRITE_THROUGH_KEYS,
sourceId: string,
opts: { managed: boolean; dryRun: boolean },
): Promise<DerivedWriteThrough | null> {
if (opts.managed || opts.dryRun || !await derivedWriteThroughEnabled(engine, phase)) return null;
const held: string[] = [];
const failed: DerivedCommitFailure[] = [];
const kept = (skipped: string | undefined) => skipped === 'file_database_drift' || skipped === 'file_changed';
const write = async (slugs: readonly string[], retired: readonly string[] = []) => {
if (failed.length) return markRetry(engine, [...slugs, ...retired].map(slug => ({ sourceId, slug })));
const changes: DerivedFileChange[] = [];
for (const slug of slugs) {
const result = await writeDerivedFile(engine, slug, sourceId, {});
if (result.change) changes.push(result.change);
if (kept(result.skipped)) held.push(fileDatabaseDriftToken(slug, sourceId, 'before', 'the generated page was not written to its file'));
}
for (const slug of retired) {
const result = await deleteDerivedFile(engine, slug, sourceId);
if (result.change) changes.push(result.change);
if (kept(result.skipped)) held.push(fileDatabaseDriftToken(slug, sourceId, 'before', 'the retired page\'s file was not removed'));
}
const failures = await commitDerivedChanges(engine, phase, changes);
failed.push(...failures);
const unsettled = failedChanges(changes, failures);
await markRetry(engine, unsettled);
await clearRetry(engine, sourceId, [...slugs, ...retired].filter(slug => !unsettled.some(c => c.slug === slug)));
};
const rows = await engine.executeRaw<{ slug: string; retired: boolean }>(
`SELECT slug, deleted_at IS NOT NULL AS retired FROM pages p
WHERE source_id = $1
AND ((deleted_at IS NULL AND source_path IS NULL)
OR EXISTS (SELECT 1 FROM op_checkpoints c WHERE c.op = $4 AND c.fingerprint = p.source_id || ':' || p.slug))
AND type = $2 AND ($2 <> 'concept' OR frontmatter->>'synthesized_by' LIKE 'synthesize_concepts%')
AND COALESCE(frontmatter->>'source_hash', '') NOT LIKE 'pending:%'
ORDER BY id LIMIT $3`,
[sourceId, phase === 'extract_atoms' ? 'atom' : 'concept', DERIVED_BACKFILL_LIMIT, RETRY_OP],
);
await write(rows.filter(r => !r.retired).map(r => r.slug), rows.filter(r => r.retired).map(r => r.slug));
return { write, held, failed };
}
/** What a dream reverse-write did with one page's file. `held` is its `file_database_drift` entry. */
export type DreamPageFileWrite = { written: true; path: string } | { written: false; held?: string; skipped?: string; error?: string };
/** The file of a dream page: its record target (`resolvePageWriteTarget`), else the caller's brainDir layout, unless brainDir is the native source's read-only checkout (#8935). */
async function dreamFileTarget(engine: BrainEngine, brainDir: string, slug: string, sourceId: string, nativeSourceId: string):
Promise<{ filePath: string; root: string } | { skipped: string }> {
const target = await resolvePageWriteTarget(engine, slug, sourceId);
if (!target.ok && (!brainDir || (target.skipped !== 'no_repo_configured' && target.skipped !== 'repo_not_found'))) return { skipped: target.skipped };
if (target.ok) return { filePath: target.filePath, root: target.writeRoot };
if (sourceId !== nativeSourceId && await readOnlyCheckout(engine, nativeSourceId)) return { skipped: 'read_only_checkout' };
return { filePath: sourceId === nativeSourceId ? join(brainDir, `${slug}.md`) : join(brainDir, '.sources', sourceId, `${slug}.md`), root: brainDir };
}
/**
* #6982: write the file of a page the dream cycle wrote in the database (a synthesis or patterns output, the cycle
* summary) without replacing an edit only the file holds, then commit it (#8073: `commitDerivedChanges`; a refused
* commit rolls the file back and returns its `write_through_failed` hold). Nothing is written with `sync.write_through`
* off, for a read-only mirror or a clone gbrain owns (#8935: `readOnlyCheckout`, both checked before any path is
* resolved, so a mirror without a checkout gets no brainDir fallback file either), or for a declared `storage.db_only`
* slug: the database is the copy of record. The file is
* the page's file of record (`resolvePageWriteTarget`: recorded source_path, own-local_path roots); a source with no
* usable checkout (no root configured, or the root is missing) keeps the caller's brainDir layout (`<brainDir>/<slug>.md`,
* foreign sources under `.sources/<id>/`). The file is replaced only when `replaceableFile` finds nothing in it the
* database lacks, and is re-read after the replacement is staged, right before the rename, so an edit that lands in
* between is kept as well.
*
* This is the reverse-write's own direct write (a staged sibling renamed over the file), as before, not
* `writePageThrough` or `atomicWriteFileSync`: once a subagent's tool write has gone through the coordinator, the
* checkout carries an owner marker and both refuse `writer_coordinator_required`, which would leave the file behind the
* database for the next coordinated write to refuse as drift. Such a checkout commits through commit-push's named-paths
* path instead of the classic door.
*/
export async function writeDreamPageFile(engine: BrainEngine, brainDir: string, slug: string, sourceId: string,
opts: { nativeSourceId: string; markdown: string }): Promise<DreamPageFileWrite> {
const file = await writeDreamFile(engine, brainDir, slug, sourceId, opts);
if (!file.change) return file.result;
const [failure] = await commitDerivedChanges(engine, 'dream', [file.change]);
return failure ? { written: false, held: dreamCommitFailedToken(slug, failure) } : file.result;
}
/** The file write of `writeDreamPageFile`, uncommitted, with the change it made. */
async function writeDreamFile(engine: BrainEngine, brainDir: string, slug: string, sourceId: string,
opts: { nativeSourceId: string; markdown: string }): Promise<{ result: DreamPageFileWrite; change?: DerivedFileChange }> {
if (await isWriteThroughDisabled(engine)) return { result: { written: false, skipped: 'disabled_by_config' } };
if (await readOnlyCheckout(engine, sourceId)) return { result: { written: false, skipped: 'read_only_checkout' } };
const target = await dreamFileTarget(engine, brainDir, slug, sourceId, opts.nativeSourceId);
if ('skipped' in target) return { result: { written: false, skipped: target.skipped } };
try {
const storage = loadStorageConfig(target.root);
if (storage && isDbOnly(slug, storage)) return { result: { written: false, skipped: 'db_only' } };
} catch { /* unreadable gbrain.yml: sync reports it */ }
const { filePath } = target;
const check = await replaceableFile(engine, sourceId, slug, filePath);
if (!check.ok) return { result: { written: false, held: dreamFileDriftToken(slug, sourceId, 'before') } };
mkdirSync(dirname(filePath), { recursive: true });
const staged = `${filePath}.${process.pid}.tmp`;
writeFileSync(staged, opts.markdown, 'utf-8');
if ((existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null) !== check.expectedFileBytes) {
rmSync(staged, { force: true });
return { result: { written: false, held: dreamFileDriftToken(slug, sourceId, 'during') } };
}
renameSync(staged, filePath);
noteWritten(target.root, filePath);
return { result: { written: true, path: filePath },
change: { slug, sourceId, filePath, writeRoot: target.root, prior: check.expectedFileBytes, after: opts.markdown, bound: null } };
}
/**
* GBRA-161 PR 4: remove the file of a dream page a generation retired, resolved the way `writeDreamPageFile` wrote it, only
* when the retired page (or one of its versions) holds everything in it; re-read right before the unlink. A file holding
* an edit the database lacks stays, with its `file_database_drift` entry returned as `held`. The removal commits
* (#8073); a refused commit puts the file back and returns its `write_through_failed` hold. A read-only mirror or a
* clone gbrain owns keeps the file (#8935).
*/
export async function removeDreamPageFile(engine: BrainEngine, brainDir: string, slug: string, sourceId: string,
nativeSourceId: string): Promise<{ removed: boolean; held?: string }> {
if (await isWriteThroughDisabled(engine) || await readOnlyCheckout(engine, sourceId)) return { removed: false };
const target = await dreamFileTarget(engine, brainDir, slug, sourceId, nativeSourceId);
if ('skipped' in target) return { removed: false };
const { filePath } = target;
if (!existsSync(filePath)) return { removed: false };
const check = await replaceableFile(engine, sourceId, slug, filePath);
if (!check.ok || readFileSync(filePath, 'utf-8') !== check.expectedFileBytes) {
return { removed: false, held: fileDatabaseDriftToken(slug, sourceId, 'before', "the retired dream page's file was not removed") };
}
rmSync(filePath, { force: true });
noteWritten(target.root, filePath);
const [failure] = await commitDerivedChanges(engine, 'dream retirement',
[{ slug, sourceId, filePath, writeRoot: target.root, prior: check.expectedFileBytes, after: null, bound: null }]);
return failure ? { removed: false, held: dreamCommitFailedToken(slug, failure) } : { removed: true };
}
/** #6982: the hold entry for a dream page whose file holds an edit the database lacks. */
function dreamFileDriftToken(slug: string, sourceId: string, when: 'before' | 'during'): string {
return fileDatabaseDriftToken(slug, sourceId, when, 'the dream output stays in the database only', 'the dream reverse-write');
}
/** #8073: the hold entry for a dream page whose file commit the checkout refused (the file was rolled back). */
function dreamCommitFailedToken(slug: string, failure: DerivedCommitFailure): string {
return `${slug} (write_through_failed: ${failure.code}: the checkout refused the commit of this page's file, so the file was rolled back `
+ `and the dream output stays in the database only; ${failure.kept.length ? `${failure.kept.length} file(s) changed meanwhile were left as they are; ` : ''}`
+ 'fix the checkout (git status there), then the next write of the page commits its file)';
}
/**
* The dream reverse-write loop (synthesize, patterns, #6982): each ref's page rendered by `render` and written through
* `writeDreamFile`, then the files written commit as one batch (#8073), also when an abort or an error stops the loop. A
* page whose file holds an edit the database lacks keeps that file and its `file_database_drift` entry is pushed to
* `held`; a refused commit rolls the batch back and pushes each page's `write_through_failed` entry; an abort stops
* before the next file; other per-page failures are logged and skipped. Returns the number of files written and committed.
*/
export async function reverseWriteDreamRefs<R extends { slug: string; source_id: string }>(engine: BrainEngine, brainDir: string, refs: R[],
opts: { phase: string; nativeSourceId: string; render: (page: Page, tags: string[], ref: R) => string; signal?: AbortSignal; held?: string[] }): Promise<number> {
const changes: DerivedFileChange[] = [];
let rolledBack = 0;
try {
for (const ref of refs) {
const { slug, source_id } = ref;
throwIfAborted(opts.signal, `[dream] ${opts.phase} reverse-write`);
// v0.32.8 F6: validate source_id is filesystem-safe before any join().
validateSourceId(source_id);
const page = await engine.getPage(slug, { sourceId: source_id });
if (!page) continue;
const tags = await engine.getTags(slug, { sourceId: source_id });
// #4077: re-check after the row reads — an abort that lands during
// getPage/getTags must not reach this ref's file write.
throwIfAborted(opts.signal, `[dream] ${opts.phase} reverse-write`);
try {
const file = await writeDreamFile(engine, brainDir, slug, source_id, { nativeSourceId: opts.nativeSourceId, markdown: opts.render(page, tags, ref) });
if (file.change) changes.push(file.change);
else if (!file.result.written && file.result.held) opts.held?.push(file.result.held);
else if (!file.result.written && file.result.error) process.stderr.write(`[dream] reverse-write ${slug}@${source_id} failed: ${file.result.error}\n`);
} catch (e) {
// Per-slug failures are non-fatal — phase continues.
process.stderr.write(`[dream] reverse-write ${slug}@${source_id} failed: ${e instanceof Error ? e.message : String(e)}\n`);
}
}
} finally {
for (const failure of await commitDerivedChanges(engine, opts.phase, changes)) {
for (const change of changes.filter(c => c.writeRoot === failure.root && failure.paths.includes(relative(c.writeRoot, c.filePath)))) {
opts.held?.push(dreamCommitFailedToken(change.slug, failure));
rolledBack++;
}
}
}
return changes.length - rolledBack;
}