Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Convergence PR B (docs/designs/REFRESH_WRITE_BARRIER.md): the replay side of the refresh write barrier.
*
* A page write admitted while `gbrain sources refresh` held its worktree waits in state `deferred` (journal.ts) until the
* refresh is released (completed or aborted). When it is claimed, before anything is published, two checks run here:
* the write's durable authority against current grants (it may have waited minutes), and whether the refresh changed
* any file the publication writes. A refresh that changed one ends the write `revision_conflict` (nothing applied), so
* a page the refresh changed is never overwritten. The check reads git, not the page revision: a refresh completes even
* when a member's sync is blocked, so git can hold the refreshed bytes before the database does.
*/
import { relative, sep } from 'node:path';
import { execFileBounded } from '../bounded-child-exec.ts';
import { gitChildEnv } from '../git-env.ts';
import { OperationError } from '../ops/contract.ts';
import { authorizeWrite } from './authority.ts';
import type { BrainEngine } from '../engine.ts';
import { completeWrite, SYNC_KIND } from './journal.ts';
import { PENDING_IMPORT } from './pending-import.ts';
import { declarePersistenceProtocol } from './protocol.ts';
import type { SqlEngine, WriteRequest } from './model.ts';
const GIT_ENV = { GIT_TERMINAL_PROMPT: '0', GCM_INTERACTIVE: 'Never', LC_ALL: 'C', GIT_OPTIONAL_LOCKS: '0' };
const DIFF_TIMEOUT_MS = 30_000;
const CACHED_REFRESHES = 32;
/** The paths each refresh changed, per checkout root: one diff serves the whole drain behind it. */
const changed = new Map<string, Promise<RefreshPaths | null>>();
/** GST-1623: with `core.ignorecase` true the checkout names one file in any case, so paths are compared case-folded. */
interface RefreshPaths { paths: ReadonlySet<string>; fold: boolean }
/** Paths (repository-relative, forward slashes) refresh `refreshId` changed; empty for an aborted refresh; null when unreadable. */
async function refreshChangedPaths(engine: SqlEngine, refreshId: string, root: string): Promise<RefreshPaths | null> {
const key = `${refreshId}\0${root}`;
const held = changed.get(key);
if (held) return held;
const read = (async () => {
const [refresh] = await engine.executeRaw<{ state: string; old_head: string; target_head: string; released: boolean }>(
'SELECT state,old_head,target_head,released_at IS NOT NULL AS released FROM persistence_worktree_refreshes WHERE id=$1::uuid', [refreshId]);
if (!refresh?.released) return null;
if (refresh.state === 'aborted' || refresh.old_head === refresh.target_head) return { paths: new Set<string>(), fold: false };
const [diff, config] = await Promise.all([execFileBounded('git', ['-C', root, '-c', 'core.quotepath=false', 'diff', '--name-only', '-z', '--no-renames', '--relative',
refresh.old_head, refresh.target_head], { timeout: DIFF_TIMEOUT_MS, maxBuffer: 64 * 1024 ** 2, env: gitChildEnv(GIT_ENV) }),
execFileBounded('git', ['-C', root, 'config', '--bool', 'core.ignorecase'], { timeout: DIFF_TIMEOUT_MS, env: gitChildEnv(GIT_ENV) })]);
if (diff.error) return null;
const fold = !config.error && config.stdout.trim() === 'true';
return { paths: new Set(diff.stdout.split('\0').filter(Boolean).map(path => fold ? path.toLowerCase() : path)), fold };
})();
changed.set(key, read);
read.then(paths => { if (paths === null && changed.get(key) === read) changed.delete(key); }, () => { if (changed.get(key) === read) changed.delete(key); });
while (changed.size > CACHED_REFRESHES) changed.delete(changed.keys().next().value!);
return read;
}
export interface DeferredRefusal { state: 'conflict' | 'failed'; failure: { code: string; message: string } }
/** Writes that apply on top of the page as it is, so one accepted later never makes them overwrite anything. */
const APPENDING_OPERATIONS = ['remember', 'add_timeline_entry'];
/**
* GST-1623: the refusal of deferred write `row` when a write to its page accepted after it has already committed (an older
* consumer, which has no EARLIER_DEFERRED in its claim, claimed the later write first), so the deferred one, replayed now,
* never lands on top of it. Managed sync's own rows are the import the write waited for and do not count; neither does a
* write that only appends. A sync row is a `submit_job` with a managed_sync_ kind, or with an imported file once compaction
* dropped its intent (#8765 review): a caller cannot make its own write look like one. A connector import names
* `connector_database` in its outcome and stays a later write (overwritten-edits.ts MANAGED_IMPORT).
*/
async function laterWriteRefusal(engine: SqlEngine, row: WriteRequest): Promise<DeferredRefusal | null> {
if (APPENDING_OPERATIONS.includes(row.operation)) return null;
const [later] = await engine.executeRaw<{ request_id: string }>(`SELECT r.request_id::text AS request_id FROM persistence_requests r
WHERE r.source_id=$1 AND r.slug=$2 AND r.sequence>$3 AND r.state='committed' AND NOT (r.operation='submit_job'
AND (${SYNC_KIND('r')} OR (r.intent IS NULL AND COALESCE(r.outcome->>'imported_file','')='true' AND NOT (r.outcome ? 'connector_database')))) ORDER BY r.sequence LIMIT 1`,
[row.source_id, row.slug, String(row.sequence)]);
if (!later) return null;
return { state: 'conflict', failure: { code: 'revision_conflict', message: `Request ${row.request_id} waited ${row.deferred_refresh_id ? `behind refresh ${row.deferred_refresh_id}` : 'for an import'}, and request ${later.request_id}, accepted after it, has already changed ${row.slug}, so nothing was applied and the later write was not overwritten. Read the page again and submit the change with a new request_id.` } };
}
/**
* Null when a claimed deferred write may publish `files` (its own page file and its companions'); otherwise the terminal
* refusal to record. A row that was never deferred is not checked. GST-1623: a write held for an import is checked too, for a
* later write to its page that committed first.
*/
export async function deferredReplayRefusal(engine: SqlEngine, row: WriteRequest, files: ReadonlyArray<{ path: string; root: string }>): Promise<DeferredRefusal | null> {
const refreshId = row.deferred_refresh_id;
if (!refreshId && row.deferred_import_from == null) return null;
const later = await laterWriteRefusal(engine, row);
if (later || !refreshId) return later;
try { await authorizeWrite(engine, row.authority, row.operation, row.slug); } catch (error) {
if (!(error instanceof OperationError)) throw error;
return { state: 'failed', failure: { code: error.code, message: `Request ${row.request_id} waited behind refresh ${refreshId}, and its credential no longer grants this write: ${error.message}` } };
}
let unreadable = false;
const touched: string[] = [];
for (const file of files) {
const changes = await refreshChangedPaths(engine, refreshId, file.root);
const target = relative(file.root, file.path).split(sep).join('/');
if (changes === null) unreadable = true;
if (changes === null || changes.paths.has(changes.fold ? target.toLowerCase() : target)) touched.push(target);
}
if (!touched.length) return null;
return { state: 'conflict', failure: { code: 'revision_conflict', message: unreadable
? `Request ${row.request_id} waited behind refresh ${refreshId}, and what that refresh changed could not be read, so nothing was applied. Read the page again and submit the change with a new request_id.`
: `Refresh ${refreshId} changed ${touched.join(', ')} after request ${row.request_id} was accepted, so nothing was applied and no newer content was overwritten. Read the page again and submit the change with a new request_id.` } };
}
export const REFRESH_BARRIER_MAX_WAIT_KEY = 'persistence.refresh_barrier_max_wait_ms';
export const REFRESH_BARRIER_MAX_WAIT_MS = 15 * 60_000;
/** The wait limit in ms: the configured value when it is a positive integer, else the default. */
const MAX_WAIT_SQL = `COALESCE((SELECT CASE WHEN value ~ '^[1-9][0-9]{0,14}$' THEN value::double precision END FROM config WHERE key='${REFRESH_BARRIER_MAX_WAIT_KEY}'), ${REFRESH_BARRIER_MAX_WAIT_MS})`;
const overdueDeferredFrom = (host: string, maxWait: string) => `JOIN persistence_worktrees w ON w.id=r.worktree_id
JOIN persistence_worktree_refreshes f ON f.id=r.deferred_refresh_id
WHERE w.owner_host_id=${host}::uuid AND r.state='deferred' AND f.released_at IS NULL AND r.created_at<now()-(${maxWait}*interval '1 millisecond')`;
/**
* GST-1623: however often a write is held again (another writer committing to its file faster than sync imports), it waits at
* most this many wait limits from its admission, and never less than HOLD_CEILING_FLOOR_MS.
*/
const HOLD_CEILING_LIMITS = 4;
const HOLD_CEILING_FLOOR_MS = 4 * 60 * 60_000;
const holdCeiling = (maxWait: string) => `GREATEST(${maxWait}*${HOLD_CEILING_LIMITS},${HOLD_CEILING_FLOOR_MS})`;
/** A held write past its limit: held longer than the wait limit, or accepted longer ago than the ceiling. */
const holdOverdue = (maxWait: string) => `(r.updated_at<now()-(${maxWait}*interval '1 millisecond') OR r.created_at<now()-(${holdCeiling(maxWait)}*interval '1 millisecond'))`;
/** GST-1582: a write held for an import still pending (its refresh, if any, released) whose hold is past its limit. */
const overdueImportFrom = (host: string, maxWait: string) => `JOIN persistence_worktrees w ON w.id=r.worktree_id JOIN sources s ON s.id=r.source_id
WHERE w.owner_host_id=${host}::uuid AND r.state='deferred' AND r.deferred_import_from IS NOT NULL AND COALESCE(s.last_commit,'')=r.deferred_import_from
AND (r.deferred_refresh_id IS NULL OR EXISTS (SELECT 1 FROM persistence_worktree_refreshes f WHERE f.id=r.deferred_refresh_id AND f.released_at IS NOT NULL))
AND ${holdOverdue(maxWait)}`;
/** The owner's idle probe counts an overdue deferred or import-held write as work, so its expiry runs even when nothing else is queued. */
export const overdueDeferredWorkSql = (host: string) => `(EXISTS (SELECT 1 FROM persistence_requests r ${overdueDeferredFrom(host, MAX_WAIT_SQL)})
OR EXISTS (SELECT 1 FROM persistence_requests r ${overdueImportFrom(host, MAX_WAIT_SQL)}))`;
/**
* The owner's housekeeping: a deferred write whose refresh is still unreleased past the wait limit ends failed with
* `refresh_barrier_expired` (nothing applied, counters released). The transaction declares the persistence protocol (the
* brain row FOR SHARE) before it locks the refresh row, the order fenceAndMerge takes them in (brain FOR UPDATE, then the
* refresh), so an expiry overlapping a fence waits instead of deadlocking; with the refresh row locked, an expiry and
* the release that would make the row claimable never both win.
* GST-1582: one held for an import that has not landed within the limit of its hold ends `pending_import_expired` the
* same way, except a `remember`, which ends as its `pending_import` refusal did (conflict `source_changed`, origin
* `pending_import`), so the first sync that imports the file replays the fact. Its claim and its expiry both move the row,
* so the expiry checks the row is still unclaimed and its import still pending with the sources row locked FOR SHARE
* (after the protocol's brain lock, the order a fence takes them in), so an import that is landing makes the check wait
* and then see it landed. GST-1623: it also rechecks the hold it scanned (the same import and an age past the limit), so an
* import and a new hold between the scan and the lock never expire the new hold. A write held again and again expires at
* the ceiling counted from its admission (holdOverdue). An expired remember says whether a sync will replay it: only one that named its entity is replayed (failed-writes.ts).
*/
export async function expireDeferredWrites(engine: BrainEngine, hostId: string, signal?: AbortSignal): Promise<WriteRequest[]> {
const [config] = await engine.executeRaw<{ ms: number }>(`SELECT ${MAX_WAIT_SQL} AS ms`, undefined, { signal });
const maxWaitMs = Number(config?.ms ?? REFRESH_BARRIER_MAX_WAIT_MS);
const expired = await engine.executeRaw<WriteRequest>(`SELECT r.* FROM persistence_requests r ${overdueDeferredFrom('$1', '$2::double precision')}
ORDER BY r.sequence LIMIT 100`, [hostId, maxWaitMs], { signal });
const finished: WriteRequest[] = [];
for (const row of expired) {
const done = await engine.transaction(async tx => {
await declarePersistenceProtocol(tx);
const [held] = await tx.executeRaw('SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NULL FOR NO KEY UPDATE', [row.deferred_refresh_id]);
if (!held) return null;
return completeWrite(tx, row, 'failed', {}, { code: 'refresh_barrier_expired',
message: `Request ${row.request_id} waited behind refresh ${row.deferred_refresh_id} for more than ${Math.round(maxWaitMs / 1000)} s, and the refresh has not finished, so nothing was applied. Check the refresh with gbrain sources writer status --source ${row.source_id} --json, then submit the write again with a new request_id.` });
});
if (done) finished.push(done);
}
const imports = await engine.executeRaw<WriteRequest>(`SELECT r.* FROM persistence_requests r ${overdueImportFrom('$1', '$2::double precision')}
ORDER BY r.sequence LIMIT 100`, [hostId, maxWaitMs], { signal });
await afterImportScan?.(imports);
for (const row of imports) {
const done = await engine.transaction(async tx => {
await declarePersistenceProtocol(tx);
const [held] = await tx.executeRaw(`SELECT 1 FROM persistence_requests r JOIN sources s ON s.id=r.source_id
WHERE r.id=$1::uuid AND r.state='deferred' AND r.deferred_import_from=$2 AND COALESCE(s.last_commit,'')=r.deferred_import_from
AND ${holdOverdue('$3::double precision')} FOR SHARE OF s`, [row.id, row.deferred_import_from, maxWaitMs]);
if (!held) return null;
const waited = `Request ${row.request_id} waited more than ${Math.round(maxWaitMs / 1000)} s`;
const status = `gbrain sync status --source ${row.source_id} --json`;
if (row.operation === 'remember') return completeWrite(tx, row, 'conflict', {}, { code: 'source_changed', detail: { origin: PENDING_IMPORT },
message: `${waited} for managed sync to import a committed change to the file of ${row.slug}, and the import has not landed, so the fact is not saved yet. ${typeof row.intent?.entity_slug === 'string'
? `It is kept for one automatic replay attempt by the first managed sync that imports the file; until that replay commits, gbrain repair failed-writes --source ${row.source_id} lists it (a replay that is refused stays there for a person to replay).`
: `It is kept, but it named no entity, so no sync replays it: gbrain repair failed-writes --source ${row.source_id} lists it for a person to replay once the source has synced.`} Do not save it again; check the source with ${status}.` });
return completeWrite(tx, row, 'failed', {}, { code: 'pending_import_expired',
message: `${waited} for managed sync to import a committed change to the file of ${row.slug}, and the import has not landed, so nothing was applied. Check the source with ${status}, then submit the write again with a new request_id.` });
}).catch(error => { if ((error as { code?: string })?.code === 'write_claim_lost') return null; throw error; });
if (done) finished.push(done);
}
return finished;
}
/**
* Whether a deferred write still waits: behind a refresh not yet released, or (GST-1582) for an import that has not landed
* (`sources.last_commit` still the commit it was held at). A read that fails counts as waiting.
*/
export async function deferredWaiting(engine: SqlEngine, row: Pick<WriteRequest, 'source_id' | 'deferred_refresh_id' | 'deferred_import_from'>): Promise<boolean> {
const [held] = await engine.executeRaw<{ waiting: boolean }>(`SELECT
($1::uuid IS NOT NULL AND NOT EXISTS (SELECT 1 FROM persistence_worktree_refreshes WHERE id=$1::uuid AND released_at IS NOT NULL))
OR ($2::text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM sources WHERE id=$3 AND COALESCE(last_commit,'')<>$2::text)) AS waiting`,
[row.deferred_refresh_id ?? null, row.deferred_import_from ?? null, row.source_id]).catch(() => [{ waiting: true }]);
return held?.waiting !== false;
}
/** Writes admitted deferred that have not finished (waiting, or claimed and still publishing), for `gbrain upgrade --check-downgrade`. */
export async function deferredBacklog(engine: SqlEngine): Promise<{ count: number; oldest_age_ms: number }> {
const [row] = await engine.executeRaw<{ count: number; oldest_age_ms: number | null }>(`SELECT count(*)::int AS count,
(EXTRACT(EPOCH FROM now()-min(created_at))*1000)::bigint AS oldest_age_ms FROM persistence_requests
WHERE (deferred_refresh_id IS NOT NULL OR deferred_import_from IS NOT NULL) AND state IN ('deferred','queued','running','recovering')`);
return { count: Number(row?.count ?? 0), oldest_age_ms: Number(row?.oldest_age_ms ?? 0) };
}
let afterImportScan: ((rows: WriteRequest[]) => Promise<void>) | null = null;
/** Test seam: runs between the expiry's scan of import holds and their transactions. */
export function _setAfterImportExpiryScanForTests(fn: typeof afterImportScan): void { afterImportScan = fn; }
/** Test seam: forget every cached refresh diff. */
export function _resetRefreshBarrierCacheForTests(): void { changed.clear(); }