Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { hasSourceFilesystemLock, withSourceFilesystemLock, assertSourceFilesystemActive } from './minions/source-filesystem.ts';
/**
* Shared disk write-through for the canonical ingestion path.
*
* After a page row lands in the DB (via importFromContent / putPage), this
* renders the row to markdown via `serializePageToMarkdown` and writes it to
* `sync.repo_path` so the brain repo has a committable `.md` artifact that
* round-trips cleanly through `gbrain sync`. The file is rendered FROM the DB
* row, so the two sinks cannot diverge.
*
* Extracted from the v0.38 `put_page` write-through (operations.ts) so the
* `put_page` op AND `gbrain brainstorm/lsd --save` share one implementation
* instead of hand-rolling parallel (and divergent) copies. The extraction also
* upgraded the write to be ATOMIC — the original used a bare `writeFileSync`
* into a live git tree that `gbrain sync` / autopilot actively walk, so a crash
* mid-write left a partial `.md` that sync would fail to parse. We now write to
* a unique temp sibling and `rename` into place (rename is atomic on the same
* filesystem), matching the `.tmp + rename` convention used by
* import-checkpoint.ts / op-checkpoint.ts.
*
* Trust gating (subagent sandbox, dry-run) stays at the CALLER — this helper
* only does "row exists + repo is a real dir → render + atomic write".
*/
import { existsSync, statSync, mkdirSync, unlinkSync, readdirSync, readFileSync, renameSync, rmSync } from 'fs';
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'path';
import { atomicStagingPath, atomicWriteFileSync } from './atomic-write.ts';
import { flushDirectory } from './fs-durable.ts';
import { isSegmentedPage } from './page-body/page-shape.ts';
import type { BrainEngine } from './engine.ts';
import type { StreamedFileContent } from './persistence/large-file.ts';
import { isRelativeFileUri, serializePageToMarkdown, resolvePageFilePath, resolveSourceLocalFilePath } from './markdown.ts';
import { isForeignAbsolutePath, isHostAbsolutePath, isWriteTargetContained, msysToNativePath } from './path-confine.ts';
import { readSlugRootMode, resolveSlugRootMode, type SlugRootMode } from './sync-anchor.ts';
import { resolveSlugForPath } from './sync.ts';
import { PUSH_SCAN_MAX_FILE_BYTES } from './page-size-policy.ts';
import { canonicalFilesystemPath } from './persistence/root-registry.ts';
import {
isDurabilityHardened, commitWriteThroughFile, currentBranch, getLastPushOutcome,
type PushLogOutcome,
} from './brain-repo-durability.ts';
import { bindOrigin } from './persistence/origin-guard.ts';
import { readOnlyCheckout } from './persistence/mirror-read-only.ts';
/** Minimal logger surface — structurally compatible with operations.ts `Logger`. */
export interface WriteThroughLogger {
warn(msg: string): void;
}
export interface WriteThroughResult {
written: boolean;
path?: string;
/** The bytes written to `path` (set with `written: true` for an inline page; a page stored in segments is streamed and never held whole), so a caller can tell its own write from a later edit. */
content?: string;
/**
* True when the write was also committed to git (#2426). Only attempted on
* repos hardened via `gbrain sources harden` (durability hook installed).
* Commit-only — this says nothing about whether the commit ever reached the
* remote. Best-effort — a false/absent value never blocks the write.
*/
committed?: boolean;
/**
* Set alongside `committed: true`. The actual push runs detached in the
* post-commit hook (see brain-repo-durability.ts), so at the moment this
* result is returned the outcome for THIS commit is genuinely unknown —
* 'pending' is the only honest value. Check `lastPushStatus` (or
* $GBRAIN_HOME/brain-push.log directly) afterward to see whether pushes for
* this branch are landing.
*/
pushed?: 'pending';
/**
* Best-effort snapshot of the most recently logged push outcome for this
* branch (read from the hook's shared log), taken right after the commit
* above. It reflects push history UP TO that point — not the push this
* write just queued — so callers and health tooling can tell "pushes for
* this branch have been failing" apart from "this write committed fine".
*/
lastPushStatus?: PushLogOutcome;
/**
* Non-error reasons the file was not written:
* - disabled_by_config: `sync.write_through` is set to an off value
* ('false'/'0'/'off'/'no', case-insensitive) — the brain is DB-only by
* operator choice (e.g. the host repo is a shared working tree where
* stray root-level `.md` artifacts are unwanted). Checked before any FS
* or DB work.
* - no_repo_configured: the resolved target (source `local_path` or, for a
* sole-source brain, `sync.repo_path`) is unset (DB-only by design).
* - repo_not_found: target set but relative (#7630), missing, or not a directory.
* - source_repo_belongs_to_other_source: the assigned source has no
* `local_path`, and `sync.repo_path` is another source's own working tree
* — #2018: writing here would pollute that sibling's repo, so we skip.
* - page_not_found_after_write: the DB row isn't readable back (the caller's
* DB write failed or targeted a different source).
* - path_escapes_source_root: the computed file path resolves outside the
* source's working tree (hostile slug row / symlinked subtree) — refused.
* - case_insensitive_collision: on a case-insensitive filesystem
* (macOS/Windows default), the target directory already holds a
* differently-cased entry that the FS folds onto this page's file, so
* writing would silently clobber the OTHER slug's file (#2831) — refused.
* - file_changed: the caller passed `expectedFileBytes` and the file on
* disk no longer holds them (an edit landed after the caller read it) —
* the edit is kept and nothing is written.
* - over_push_scan_cap: the rendered file is over the pre-push secret-scan
* cap (PUSH_SCAN_MAX_FILE_BYTES), so committing it would block every push.
* - mirror_read_only / url_clone: the source is a read-only mirror (#5409,
* #7098) or a clone gbrain owns and pulls (#6946); its checkout is never written.
*/
skipped?: 'disabled_by_config' | 'mirror_read_only' | 'url_clone' | 'no_repo_configured' | 'repo_not_found' | 'source_repo_belongs_to_other_source' | 'page_not_found_after_write' | 'path_escapes_source_root' | 'case_insensitive_collision' | 'file_changed' | 'over_push_scan_cap' | 'segmented_page';
/** Caller-visible advisory when a permitted DB-only outcome is still risky. */
warning?: string;
/** Set when the render/write/rename itself threw (EACCES, ENOTDIR, disk full). */
error?: string;
}
export interface WritePageThroughOpts {
sourceId?: string;
/** Merged over the page's own frontmatter at render time (e.g. provenance). */
frontmatterOverrides?: Record<string, unknown>;
/** Skip a page stored in segments (`segmented_page`): the caller compares the written file as a string (derived writes). */
inlineOnly?: boolean;
logger?: WriteThroughLogger;
/**
* The file bytes the caller read before writing the database row (null:
* the file was absent). Rechecked under the source filesystem lock right
* before the rename; a file that changed since is left untouched
* (`skipped: 'file_changed'`). Omitted: no preimage check.
*/
expectedFileBytes?: string | null;
/** The caller commits the file itself (#8073: one commit per batch), so the hardened per-file commit is skipped. */
deferCommit?: boolean;
/** GST-2468: the exact file to write instead of the page rendered from its row (a label-only write keeps the file's own bytes). Inline pages only. */
fileText?: string;
}
export function withNoRepoWriteThroughWarning<T extends { written: boolean; skipped?: string; warning?: string }>(result: T, sourceId: string, operation = 'put_page'): T {
if (!result.written && result.skipped === 'relative_root') {
return {
...result,
warning:
`${operation} wrote only to the database for source '${sourceId}': its stored repo path is not an absolute directory on this host, so no durable markdown file was created. ` +
`Ask the operator to run gbrain sources set-path ${sourceId} <absolute-directory> before relying on the write.`,
} as T;
}
if (result.written || result.skipped !== 'no_repo_configured') return result;
return {
...result,
warning:
`${operation} wrote only to the database for source '${sourceId}': no repo/local_path is configured, so no durable markdown file was created. ` +
'Bind this MCP server/token to a git-backed source or configure source local_path/sync.repo_path before relying on the write.',
} as T;
}
/**
* Vet a `pages.source_path` before it is trusted as a write target.
*
* The column is populated from the scanner's relative path at import time, so
* the normal value is a clean repo-relative `.md` path. This rejects the shapes
* that would make `join(root, value)` unsafe or nonsensical — absolute paths,
* `..` traversal, NUL bytes, non-markdown artifacts, and blanks. Containment is
* still re-checked by `isWriteTargetContained` after the join; this is the
* cheap structural filter in front of it.
*/
export function sanitizeRecordedSourcePath(raw: string | null | undefined): string | null {
if (!raw) return null;
const value = raw.trim();
if (!value || value.includes('\0')) return null;
if (!value.toLowerCase().endsWith('.md')) return null;
if (isAbsolute(value)) return null;
if (value.split(/[\\/]/).some((segment) => segment === '..')) return null;
return value;
}
/**
* Recover a page-root-relative write target from a `file://` `source_uri`.
*
* `gbrain capture --file` records the absolute input path as `source_uri` but
* does NOT set `source_path` (that column is the file-scanner's). So a file the
* user authored INSIDE the brain repo and then captured has no file of record,
* and the slug-derived fallback would mint a twin beside the very file that was
* just read. When the recorded URI points at a path under `pageRoot`, that path
* IS the file of record — use it.
*
* Returns null for anything not a contained `.md` file so the caller falls back
* to the slug path.
*/
export function recordedPathFromFileUri(sourceUri: string | null | undefined, pageRoot: string): string | null {
if (!sourceUri || !sourceUri.startsWith('file://') || isRelativeFileUri(sourceUri)) return null;
let abs = sourceUri.slice('file://'.length).replace(/^localhost(?=\/)/i, '');
if (!abs) return null;
// Percent-decode only when it looks encoded — the CLI stores raw paths, so a
// literal '%' in a filename must not be mangled.
if (/%[0-9A-Fa-f]{2}/.test(abs)) {
try {
abs = decodeURIComponent(abs);
} catch {
return null;
}
}
if (abs.includes('\0') || !abs.toLowerCase().endsWith('.md')) return null;
if (process.platform === 'win32' && /^\/[A-Za-z]:[\\/]/.test(abs)) abs = abs.slice(1);
const rel = relative(resolve(pageRoot), resolve(abs));
if (!rel || isAbsolute(rel) || rel.split(/[\\/]/).some((segment) => segment === '..')) return null;
return rel;
}
/**
* Off-value predicate for `sync.write_through`. Mirrors the falsy-config
* convention in minions/admission.ts (`isOffValue`): an operator typing
* 'FALSE', '0', 'Off', or 'no' means OFF — an opt-out that only matches the
* exact lowercase 'false' silently stays ON.
*/
function isOffValue(v: string): boolean {
const t = v.trim().toLowerCase();
return t === 'false' || t === '0' || t === 'off' || t === 'no';
}
// ~30s per-engine cache: writePageThrough and the facts fence lane run once
// per page in bulk loops (sync, embed catch-up, dream cycle); a config SELECT
// per page for a value that changes at human speed is pure overhead.
type WriteThroughCacheEntry = { at: number; disabled: boolean };
let writeThroughCache = new WeakMap<BrainEngine, WriteThroughCacheEntry>();
const WRITE_THROUGH_CACHE_MS = 30_000;
/** Test seam: drop the cache so config changes are visible immediately. */
export function _resetWriteThroughCacheForTest(): void {
writeThroughCache = new WeakMap();
}
/**
* True when `sync.write_through` is set to an off value — the operator chose
* a DB-only brain (no `.md` artifacts, no fence files, no write-through
* commits). Fail-open to enabled: a config read error must never silently
* turn the disk sink off. Shared by writePageThrough and the facts fence
* lane (fence-write.ts) so both disk sinks honor one flag.
*/
export async function isWriteThroughDisabled(engine: BrainEngine): Promise<boolean> {
const cached = writeThroughCache.get(engine);
if (cached && Date.now() - cached.at < WRITE_THROUGH_CACHE_MS) return cached.disabled;
let disabled = false;
try {
const v = await engine.getConfig('sync.write_through');
disabled = v != null && isOffValue(v);
} catch {
disabled = false;
}
writeThroughCache.set(engine, { at: Date.now(), disabled });
return disabled;
}
/** Resolved disk target for a page's canonical markdown artifact. */
export type PageWriteTarget =
| {
ok: true;
filePath: string;
writeRoot: string;
/**
* `filePath` expressed in the file scanner's `pages.source_path`
* convention (#774: git-root-relative when the scan root sits inside a
* git repo, scan-root-relative otherwise; forward slashes). What
* writePageThrough binds onto a source_path=NULL row after the file
* materializes (#4247).
*/
sourcePathToBind: string;
}
| { ok: false; skipped: 'no_repo_configured' | 'repo_not_found' | 'source_repo_belongs_to_other_source' | 'path_escapes_source_root' };
/**
* Scanner-convention `pages.source_path` for a file under `scanRoot` (the
* root a sync of this source walks). Mirrors sync's origin rule — a scan root
* INSIDE a git repo records GIT-ROOT-relative paths under `git-root` slug mode
* (#774) and scan-root-relative paths under `source-root` mode (#4342, #5610);
* a git-root or non-git root records root-relative — and must stay the exact
* inverse of `resolveSourceLocalFilePath` (markdown.ts): delete-reconcile keys
* on this form, so a bind in the other form reads as stale or as a different
* origin while its file is still on disk.
*/
export function scannerSourcePath(scanRoot: string, filePath: string, slugRootMode: SlugRootMode = 'git-root'): string {
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- scanRoot is the operator-written sources.local_path / sync.repo_path config root; canonicalizing it here mints no fs read/write path
const absRoot = resolve(scanRoot);
let cursor = absRoot;
while (slugRootMode === 'git-root') {
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- cursor only walks UP (dirname) from the operator-config root joined with the literal '.git' segment; existsSync boolean probe, no content ever read or served
if (existsSync(join(cursor, '.git'))) {
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- filePath was proven inside writeRoot by isWriteTargetContained before the sole call site (resolvePageWriteTarget); output is an in-memory source_path string, not an fs operand
return relative(cursor, resolve(filePath)).replaceAll('\\', '/');
}
const parent = dirname(cursor);
if (parent === cursor) break;
cursor = parent;
}
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- same: containment-checked filePath relative to the operator-config root, string minting only
return relative(absRoot, resolve(filePath)).replaceAll('\\', '/');
}
/** The slug-root mode sync records origins in for this scan root: the pin, or the mode the first sync would pin. */
export async function scannerSlugRootMode(engine: BrainEngine, sourceId: string, scanRoot: string): Promise<SlugRootMode> {
const pinned = await readSlugRootMode(engine, sourceId) ?? (sourceId === 'default' ? await readSlugRootMode(engine, undefined) : null);
if (pinned) return pinned;
const scope = scannerSourcePath(scanRoot, scanRoot);
if (!scope) return 'git-root';
return resolveSlugRootMode(engine, { sourceId, explicitGitRoot: false, slugPrefix: resolveSlugForPath(`${scope}/x.md`).slice(0, -2), dryRun: true });
}
/**
* #2018/#7203: the other source whose own working tree is `path` or contains
* it, compared by filesystem identity, so a symlink, a trailing slash, or a
* macOS case or /private alias still matches. A spelling that cannot be
* resolved counts as a match: the leak guard fails closed. `relativeOwns: false`
* leaves out live relative rows, for doctor to find an absolute owner beside them.
*/
export async function otherSourceOwning(engine: BrainEngine, sourceId: string, path: string, relativeOwns = true): Promise<string | null> {
const rows = await engine.executeRaw<{ id: string; local_path: string; archived: boolean }>(
`SELECT id, local_path, archived IS TRUE AS archived FROM sources WHERE id <> $1 AND local_path IS NOT NULL AND local_path <> '' ORDER BY id`,
[sourceId],
);
const identity = (spelling: string) => { try { return canonicalFilesystemPath(msysToNativePath(spelling)); } catch { return null; } };
const target = identity(path);
return rows.find(row => {
// #7630: a live source's relative local_path names no known tree, so like any unresolvable spelling it counts as a match.
// An absolute path written by another OS (C:\ on POSIX) is not relative, the same reading sync, doctor and --clear use.
if (!isHostAbsolutePath(row.local_path) && !isForeignAbsolutePath(row.local_path)) return relativeOwns && !row.archived;
const other = identity(row.local_path);
if (target === null || other === null) return true;
const rel = relative(other, target);
return !isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`);
})?.id ?? null;
}
/**
* Compute the ONE file a (source, slug) pair lives in on disk.
*
* #2018: pick the disk target so a page is NEVER written into a different
* source's working tree. Two legitimate topologies, plus the leak guard:
* 1. The assigned source has its OWN `local_path` (a separate working
* tree) → map its recorded Git-root-relative source_path into that
* tree, including when local_path scopes a repo subdirectory.
* 2. No per-source `local_path` → nest under the host repo
* (`sync.repo_path`): default at the root, non-default under
* `.sources/<id>/` (the established multi-source layout).
* 3. LEAK GUARD: if `sync.repo_path` is literally ANOTHER source's own
* `local_path`, nesting this page there would pollute that sibling's
* git repo (the reported bug). Skip instead.
*
* Prefer the page's recorded `source_path` — the ACTUAL file this row was
* imported from — over a slug-derived name. Deriving `<slug>.md` mints a
* SECOND file beside the original whenever the on-disk name isn't the slug,
* which is the common case for a human-authored vault: `Library/People/
* Steve Jobs.md` has slug `library/people/steve-jobs`, so a later put_page
* dropped a lowercase `steve-jobs.md` twin next to it. Two artifacts, one
* row, and the newer content in whichever the caller didn't expect.
*
* It also desyncs `gbrain sync`, which keys delete-reconcile on
* `source_path` (see collectMissingSourcePaths): the twin is invisible to
* reconcile, so deleting the ORIGINAL file deletes the page even though a
* file for it is still on disk.
*
* A NULL `source_path` means the page was born via put/capture and has no
* file of record yet — the slug-derived path stays correct for those.
*
* `opts.includeDeleted`: resolve from a soft-deleted row too. The default
* reads ACTIVE rows only (a tombstone must not steer a live write), but the
* purge remediation path in `delete_page` retries the artifact removal for a
* TOMBSTONE — whose recorded `source_path` the active-row read would miss,
* falling back to the slug-derived twin and reporting a clean
* `file_not_present` while the real file survives for sync to resurrect.
*
* Shared by `writePageThrough` AND the facts fence writer (#4204): the fence
* appends to the page's file, so both writers MUST compute the identical
* path or the fence lands in a file sync never reads back and the next
* extract_facts reconcile deletes the fence-owned DB rows.
*/
export async function resolvePageWriteTarget(
engine: BrainEngine,
slug: string,
sourceId: string,
opts: { includeDeleted?: boolean } = {},
): Promise<PageWriteTarget> {
let filePath: string;
let writeRoot: string;
let scanRoot: string;
const srcRows = await engine.executeRaw<{ local_path: string | null }>(
`SELECT local_path FROM sources WHERE id = $1`,
[sourceId],
);
// gbrain#2955: heal an msys-style local_path (`/c/Users/x`, recorded by a
// Git Bash `sources add --path` on Windows) before it is joined — raw, it
// resolves to a phantom `C:\c\Users\x` and every write silently misses the
// real vault. Identity on POSIX and for already-native paths.
const rawLocalPath = srcRows[0]?.local_path ?? null;
const sourceLocalPath = rawLocalPath ? msysToNativePath(rawLocalPath) : null;
const pathRows = await engine.executeRaw<{ source_path: string | null; source_uri: string | null }>(
`SELECT source_path, source_uri FROM pages WHERE source_id = $1 AND slug = $2${opts.includeDeleted ? '' : ' AND deleted_at IS NULL'} LIMIT 1`,
[sourceId, slug],
);
const recordedPath = sanitizeRecordedSourcePath(pathRows[0]?.source_path);
const recordedUri = pathRows[0]?.source_uri ?? null;
if (sourceLocalPath) {
// #7630: a relative local_path (a legacy ".") names whatever directory gbrain runs from, never the source's tree.
if (!isHostAbsolutePath(sourceLocalPath) || !existsSync(sourceLocalPath) || !statSync(sourceLocalPath).isDirectory()) {
return { ok: false, skipped: 'repo_not_found' };
}
filePath = recordedPath
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- result passes isWriteTargetContained before any write (#4204/#4289 guard)
? resolveSourceLocalFilePath(sourceLocalPath, recordedPath, slug, await scannerSlugRootMode(engine, sourceId, sourceLocalPath)) ?? join(sourceLocalPath, `${slug}.md`)
: join(sourceLocalPath, recordedPathFromFileUri(recordedUri, sourceLocalPath) ?? `${slug}.md`); // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- result passes isWriteTargetContained before any write (#4204/#4289 guard)
writeRoot = sourceLocalPath;
scanRoot = sourceLocalPath;
} else {
const repoPath = await engine.getConfig('sync.repo_path');
if (!repoPath) {
return { ok: false, skipped: 'no_repo_configured' };
}
if (!isHostAbsolutePath(repoPath) || !existsSync(repoPath) || !statSync(repoPath).isDirectory()) {
return { ok: false, skipped: 'repo_not_found' };
}
const pageRoot = sourceId === 'default' ? repoPath : join(repoPath, '.sources', sourceId);
const knownPath = recordedPath ?? recordedPathFromFileUri(recordedUri, pageRoot);
filePath = knownPath ? join(pageRoot, knownPath) : resolvePageFilePath(repoPath, slug, sourceId);
// Leak guard: refuse a file inside some OTHER source's own working tree
// (#2018) under any spelling, whether sync.repo_path is that tree, lies
// inside it, or contains it (#7203).
if (await otherSourceOwning(engine, sourceId, filePath)) {
return { ok: false, skipped: 'source_repo_belongs_to_other_source' };
}
writeRoot = repoPath;
// pageRoot, not repoPath: a later `sources add --path <pageRoot>` scan
// walks pageRoot, so the bind must speak that scan's convention.
scanRoot = pageRoot;
}
// Defense-in-depth (#1647-slug / codex #6): confirm the computed file path
// stays within the source's working tree before any mkdir/write. validateSlug
// already rejects `..`/backslash/control/%2e in the slug at write time, so
// this guards a pre-existing hostile row or a symlinked intermediate dir
// under the source tree from escaping to an arbitrary filesystem location.
if (!isWriteTargetContained(filePath, writeRoot)) {
return { ok: false, skipped: 'path_escapes_source_root' };
}
return { ok: true, filePath, writeRoot, sourcePathToBind: scannerSourcePath(scanRoot, filePath, await scannerSlugRootMode(engine, sourceId, scanRoot)) };
}
/**
* Render the DB row for `slug` to markdown and atomically write it under
* `sync.repo_path`. Never throws — failures are reported via the result's
* `skipped` / `error` fields. put_page calls it before its transaction commits
* and rejects non-deliberate skips so a file failure rolls back the DB write.
* A read-only mirror source is skipped (`mirror_read_only`, #7098), and so is a
* clone gbrain owns on an unmanaged brain (`url_clone`, #6946).
*/
export async function writePageThrough(
engine: BrainEngine,
slug: string,
opts: WritePageThroughOpts = {},
): Promise<WriteThroughResult> {
const sourceId = opts.sourceId ?? 'default';
try {
// Opt-out flag: `sync.write_through=false` (or '0'/'off'/'no', any case)
// makes every page write DB-only, for brains whose host repo is a shared
// working tree where per-page `.md` artifacts are unwanted. Unset or any
// other value keeps the default. Memoized per engine (~30s TTL) so bulk
// loops don't pay one config SELECT per page.
if (await isWriteThroughDisabled(engine)) {
return { written: false, skipped: 'disabled_by_config' };
}
const readOnly = await readOnlyCheckout(engine, sourceId);
if (readOnly) return { written: false, skipped: readOnly };
const target = await resolvePageWriteTarget(engine, slug, sourceId);
if (!target.ok) {
return { written: false, skipped: target.skipped };
}
const { filePath, writeRoot, sourcePathToBind } = target;
if (!hasSourceFilesystemLock(writeRoot)) {
return await withSourceFilesystemLock(engine, writeRoot, () => writePageThrough(engine, slug, opts));
}
const writtenPage = await engine.getPage(slug, { sourceId });
if (!writtenPage) {
return { written: false, skipped: 'page_not_found_after_write' };
}
const segmented = isSegmentedPage(writtenPage);
if (segmented && (opts.inlineOnly || opts.fileText !== undefined)) return { written: false, skipped: 'segmented_page' };
if (segmented && opts.frontmatterOverrides) {
return { written: false, error: `page ${slug} is stored in segments, and its file is streamed as gbrain export writes it, without frontmatter overrides` };
}
const md = segmented ? null : opts.fileText ?? serializePageToMarkdown(writtenPage, await engine.getTags(slug, { sourceId }), {
frontmatterOverrides: opts.frontmatterOverrides,
});
// A segmented body alone over the cap is refused before its segments are streamed to measure the file.
const overByBody = segmented && Number((writtenPage as { body_bytes?: number | string | null }).body_bytes ?? 0) > PUSH_SCAN_MAX_FILE_BYTES;
const streamed = segmented && !overByBody ? await segmentedPageFile(engine, slug, sourceId) : null;
if (segmented && !overByBody && !streamed) return { written: false, skipped: 'page_not_found_after_write' };
// The pre-push secret scan refuses to push any file it cannot scan, so a
// file over its cap would block every later push of the brain repo.
if (overByBody || (md !== null ? Buffer.byteLength(md, 'utf-8') : streamed!.bytes) > PUSH_SCAN_MAX_FILE_BYTES) {
opts.logger?.warn(`[write-through] ${slug}: the rendered file is over the ${PUSH_SCAN_MAX_FILE_BYTES}-byte pre-push secret-scan cap — file not written (DB row is intact)`);
return { written: false, skipped: 'over_push_scan_cap' };
}
// #2831: two distinct DB slugs differing only by case (FOO vs foo) resolve
// to the SAME file on a case-insensitive filesystem — the second write
// would silently clobber the first slug's artifact. Refuse when the target
// dir holds a differently-cased entry that the FS folds onto our path:
// exact-case entry present → normal update, falls through; on a
// case-sensitive FS the variant path doesn't exist, so the guard is a
// no-op there.
const dir = dirname(filePath);
if (existsSync(dir)) {
const base = basename(filePath);
const entries = readdirSync(dir);
if (!entries.includes(base) && existsSync(filePath)) {
// The path exists on disk but no exactly-named entry does → the FS
// folded the name (case, or unicode normalization on APFS) onto a
// different slug's file.
const clash =
entries.find((e) => e.toLowerCase() === base.toLowerCase()) ?? '(normalization variant)';
opts.logger?.warn(
`[write-through] case-insensitive collision for ${slug}: '${clash}' already occupies ${filePath} — file not written (DB row is intact)`,
);
return { written: false, skipped: 'case_insensitive_collision' };
}
}
// On Bun + Windows, mkdirSync(dir, { recursive: true }) can still throw
// EEXIST when the directory already exists (POSIX no-ops it). That aborts
// the put_page / enrich / capture write-through whenever the prefix dir
// already exists, silently leaving the DB and the .md file plane out of sync.
const fileChanged = () => opts.expectedFileBytes !== undefined
&& (existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null) !== opts.expectedFileBytes;
if (fileChanged()) return { written: false, skipped: 'file_changed' };
await engine.transaction(tx => bindOrigin(tx, { sourceId, slug, sourcePath: sourcePathToBind, onlyIfUnset: true, liveOnly: true }));
assertSourceFilesystemActive();
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
assertSourceFilesystemActive();
if (fileChanged()) return { written: false, skipped: 'file_changed' };
const changedWhileStaging = new Error('file_changed');
const verify = () => { assertSourceFilesystemActive(); if (fileChanged()) throw changedWhileStaging; };
try {
if (md !== null) atomicWriteFileSync(filePath, md, { verify });
else await writeStreamedFile(engine, filePath, streamed!, verify);
} catch (error) {
if (error === changedWhileStaging) return { written: false, skipped: 'file_changed' };
throw error;
}
// #2426: on a durability-hardened repo (user ran `gbrain sources harden`),
// commit the artifact so it reaches git — pre-fix, write-through content
// stayed uncommitted forever: never pushed, `last_sync_at` frozen, and
// silently deleted by a later `sync --full` delete-reconcile. The local
// post-commit hook background-pushes the commit. Best-effort: a commit
// failure never fails the write (the DB row + file are the durable sinks).
let committed = false;
let pushed: 'pending' | undefined;
let lastPushStatus: PushLogOutcome | undefined;
try {
assertSourceFilesystemActive();
if (!opts.deferCommit && isDurabilityHardened(writeRoot)) {
committed = commitWriteThroughFile(writeRoot, filePath, slug);
if (committed) {
pushed = 'pending';
lastPushStatus = getLastPushOutcome(currentBranch(writeRoot), writeRoot);
}
}
} catch { /* best-effort */ }
return {
written: true,
path: filePath,
...(md !== null ? { content: md } : {}),
...(committed ? { committed, pushed, lastPushStatus } : {}),
};
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
opts.logger?.warn(`[write-through] failed for ${slug}: ${msg}`);
return { written: false, error: msg };
}
}
/**
* The file of a page stored in segments, with the bytes `gbrain export` writes for it: measured from the segments of
* the generation its snapshot names (unreadPageFile), as a segmented publication prepares its file, so it is sized
* before anything is bound or written. Null when the page is gone. Read on `engine` as given, so a caller inside a
* transaction (an owner repair) reads the rows it just wrote.
*/
async function segmentedPageFile(engine: BrainEngine, slug: string, sourceId: string): Promise<StreamedFileContent | null> {
const { unreadPageFile } = await import('./persistence/page-prepare.ts');
const snapshot = await engine.readPageSnapshot(slug, { sourceId });
return snapshot ? unreadPageFile(engine, snapshot) : null;
}
/**
* Streams measured file content to `filePath` through the staging writer the withdrawal mirror and restore_page use
* (stageStreamedContent: an in-progress name, renamed once whole; bytes that are not the measured ones refuse as
* `revision_conflict`), keeping an existing file's mode as atomicWriteFileSync does, then renames it over `filePath`
* after `verify`. Never held as one string, whatever its size.
*/
async function writeStreamedFile(engine: BrainEngine, filePath: string, content: StreamedFileContent, verify: () => void): Promise<void> {
const { stageStreamedContent } = await import('./persistence/large-file.ts');
let mode: number | null = null;
try { if (existsSync(filePath)) mode = statSync(filePath).mode & 0o7777; } catch { /* stat raced a delete: default mode */ }
const stage = { path: atomicStagingPath(filePath), bytes: content.bytes, hash: content.hash };
await stageStreamedContent(engine, filePath, content, stage, mode);
try {
verify();
renameSync(stage.path, filePath);
} catch (error) {
rmSync(stage.path, { force: true });
throw error;
}
flushDirectory(dirname(filePath));
}
export interface DeleteThroughResult {
/** True when an artifact existed and was unlinked. */
removed: boolean;
/** The path that was removed (or would have been). */
path?: string;
/**
* True when the removal was also committed (path-limited) because the repo
* is durability-hardened — the delete-side twin of
* `WriteThroughResult.committed`. Absent on unhardened repos and when the
* best-effort commit did not land (the unlink still stands).
*/
committed?: boolean;
/**
* Non-error reasons nothing was removed. Shares the target-resolution skip
* vocabulary (kept in lockstep via the Extract), plus:
* - disabled_by_config: the operator opted the brain out of the disk sink.
* - mirror_read_only / url_clone: the source is a read-only mirror (#7098) or a clone gbrain owns and pulls (#6946); its checkout is never written.
* - file_not_present: the page had no artifact on disk (DB-only page, or
* already removed by hand) — a clean no-op, not a failure.
* - file_changed: the caller passed `expectedFileBytes` and the file no
* longer holds them (an edit landed after the caller read it) — kept.
*/
skipped?: 'disabled_by_config' | 'mirror_read_only' | 'url_clone' | 'file_not_present' | 'file_changed' | Extract<PageWriteTarget, { ok: false }>['skipped'];
/** Set when the unlink itself threw (EACCES, EPERM, read-only mount). */
error?: string;
}
/**
* Remove the on-disk markdown artifact for `slug` — the delete-side counterpart
* to `writePageThrough` (#4022).
*
* Why this exists: `put_page`/capture write BOTH sinks (DB row + `.md` file),
* but `delete_page` used to touch only the DB. The orphaned file then outlived
* the page, and on any repo whose brain is committed on a timer (a `snapshot`
* cron, `sources harden`'s post-commit push) the deleted page's artifact gets
* committed back into git *after* deletion — so the page reappears on the next
* `gbrain sync`, silently resurrecting content the user deleted.
*
* Deliberately mirrors `writePageThrough`'s contract: never throws, reports via
* `skipped`/`error`, and resolves its target through the shared
* `resolvePageWriteTarget` so the two planes cannot disagree about which file
* backs a page. The DB row remains the durable sink — a failure here leaves a
* stale file that the next `gbrain sync` reconciles, never a lost row.
*
* `opts.target`: resolvePageWriteTarget reads the recorded `source_path` from
* ACTIVE rows only, so the delete_page op resolves the target BEFORE
* `softDeletePage` stamps `deleted_at` and passes it here. Resolving after the
* stamp would miss the recorded path, fall back to the slug-derived twin, and
* report a clean `file_not_present` no-op while the REAL artifact stayed on
* disk — the very silent-no-op class this helper exists to close.
*
* `opts.expectedFileBytes`: the file bytes the caller compared; rechecked
* under the source filesystem lock right before the unlink, and a file that
* changed since is kept (`skipped: 'file_changed'`).
*/
export async function deletePageThrough(
engine: BrainEngine,
slug: string,
opts: { sourceId?: string; logger?: WriteThroughLogger; target?: PageWriteTarget; expectedFileBytes?: string; deferCommit?: boolean } = {},
): Promise<DeleteThroughResult> {
const sourceId = opts.sourceId ?? 'default';
try {
// `sync.write_through=false` means the operator opted this brain out of
// the disk sink entirely — never unlink files gbrain does not own.
if (await isWriteThroughDisabled(engine)) {
return { removed: false, skipped: 'disabled_by_config' };
}
const readOnly = await readOnlyCheckout(engine, sourceId);
if (readOnly) return { removed: false, skipped: readOnly };
const target = opts.target ?? await resolvePageWriteTarget(engine, slug, sourceId);
if (!target.ok) return { removed: false, skipped: target.skipped };
const { filePath, writeRoot } = target;
if (!hasSourceFilesystemLock(target.writeRoot)) {
return await withSourceFilesystemLock(engine, target.writeRoot, () => deletePageThrough(engine, slug, opts));
}
if (!existsSync(filePath)) {
return { removed: false, path: filePath, skipped: 'file_not_present' };
}
if (opts.expectedFileBytes !== undefined && readFileSync(filePath, 'utf-8') !== opts.expectedFileBytes) {
return { removed: false, path: filePath, skipped: 'file_changed' };
}
assertSourceFilesystemActive();
unlinkSync(filePath);
// Mirror writePageThrough (#2426): on a durability-hardened repo, commit
// the removal (path-limited) so the post-commit hook pushes it. Pre-fix
// the deletion sat in the working tree as an uncommitted ` D` — invisible
// to commit-driven sync (the autopilot's own sync then warned "N
// uncommitted file(s) invisible to commit-driven sync") until a human
// committed it by hand. Best-effort, like the write side: a commit failure
// never fails the delete (the DB row + the unlink are the durable state).
let committed = false;
try {
if (!opts.deferCommit && isDurabilityHardened(writeRoot)) {
committed = commitWriteThroughFile(writeRoot, filePath, slug, 'delete write-through');
}
} catch { /* best-effort */ }
return { removed: true, path: filePath, ...(committed ? { committed } : {}) };
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
opts.logger?.warn(`[write-through] delete failed for ${slug}: ${msg}`);
return { removed: false, error: msg };
}
}