Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #7190: how far each source's writes have been applied, in one read, so an agent can tell whether a write
* is searchable yet without running git, SQL or a landing loop.
*
* - indexed_through.commit: `sources.last_commit`. Only a sync pass that finishes moves it: the checkpoint
* publication that writes it is reached only through the pass's stamp, so a deadline-stopped, aborted,
* selection-scoped or partial pass never advances it. Files held at that commit are counted in
* `pending.held_files`, and deletes a mass-delete cap withheld in `pending.deletes_withheld`.
* - embedded_through: `complete` when every live page of the source that is not `embed_skip` has a vector for each
* visible chunk in the active embedding column. Held and capped large pages count as not embedded. Otherwise
* `before` is the oldest `updated_at` among the pages still waiting: every page written before it is embedded.
* - committed_through_request: a `persistence_requests.sequence` at or below which every request of the source's
* current incarnation has settled (committed, conflict, failed or cancelled): one below the oldest queued, running
* or recovering request, else the journal's newest sequence. A receipt whose `sequence` is at or below it has settled.
* - pending: unsettled requests (count and oldest age), commits and changed files on this host's checkout after
* the last synced commit, held files and withheld deletes. `count` is requests + files behind + held files +
* withheld deletes.
* - writer_heartbeat: the source's drain-lease holder and its last progress (a committed sync write, a cursor
* save or a pass-row write); `stalled` after HOLDER_STALL_MS without progress. A remote caller sees the holder's
* kind and times, never its host, pid or job id.
*
* `readWriteWatermark` is the cheap subset `get_write_request` carries (indexed reads only, no git, no lease read).
* Read-only; nothing is scheduled or written.
*/
import type { BrainEngine } from '../engine.ts';
import { embeddingsDisabled } from '../embedding-disabled.ts';
import { chunkGenerationFilter } from '../search/safe-chunks.ts';
import { quoteIdentifier } from '../search/embedding-column.ts';
import { embeddingWriteTarget } from '../page-state/projections.ts';
import { budgetRowAppliesFilter } from '../page-size-policy.ts';
import { execFileBounded } from '../bounded-child-exec.ts';
import { gitChildEnv } from '../git-env.ts';
import { readGitBehind, type BehindReader } from './sync-freshness.ts';
import { countGitHolds } from './sync-holds.ts';
import { holderStalled, publicDrainHolder, readLiveDrainLease } from './sync-lease.ts';
export interface WriteWatermark {
source_id: string;
indexed_through: { commit: string | null; synced_at: string | null };
committed_through_request: number | null;
requests_pending: { count: number; oldest_age_seconds: number | null };
}
export interface AppliedWatermark extends Omit<WriteWatermark, 'requests_pending'> {
read_at: string;
embedded_through: { state: 'complete' | 'partial' | 'embeddings_off'; commit: string | null; before: string | null; unembedded_pages: number; held_pages: number };
pending: {
count: number;
oldest_age_seconds: number | null;
requests: { count: number; oldest_age_seconds: number | null };
commits_behind: { count: number; oldest_age_seconds: number | null } | null;
files_behind: number | null;
held_files: number;
deletes_withheld: number;
};
writer_heartbeat: {
holder: { kind: string; id: string | null; host: string | null; pid: number | null } | null;
heartbeat_age_seconds: number | null;
last_progress_at: string | null;
progress_age_seconds: number | null;
stalled: boolean;
};
}
/** Changed paths between `from` and HEAD of the checkout at `root`, or null when git cannot say. */
export type FilesBehindReader = (root: string, from: string) => Promise<number | null>;
const SHA = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
const ageSeconds = (at: unknown, now: number): number | null => {
if (at === null || at === undefined) return null;
const ms = at instanceof Date ? at.getTime() : Date.parse(String(at));
return Number.isFinite(ms) ? Math.max(0, Math.round((now - ms) / 1000)) : null;
};
const iso = (at: unknown): string | null => {
if (at === null || at === undefined) return null;
const ms = at instanceof Date ? at.getTime() : Date.parse(String(at));
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
};
export const readGitFilesBehind: FilesBehindReader = async (root, from) => {
if (!SHA.test(from)) return null;
const diff = await execFileBounded('git', ['-C', root, 'diff', '--name-only', '-z', `${from}..HEAD`], { timeout: 10_000, env: gitChildEnv({ GIT_OPTIONAL_LOCKS: '0' }) });
return diff.error ? null : diff.stdout.split('\0').filter(Boolean).length;
};
/** The cheap watermark `get_write_request` carries: the source row, and two reads on the request journal's indexes. */
export async function readWriteWatermark(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, now = Date.now()): Promise<WriteWatermark> {
const [row] = await engine.executeRaw<{ last_commit: string | null; last_sync_at: unknown; pending_min: number | string | null;
pending_count: number | string; pending_since: unknown }>(`SELECT s.last_commit, s.last_sync_at, p.pending_min, p.pending_count, p.pending_since
FROM sources s LEFT JOIN LATERAL (SELECT min(r.sequence) AS pending_min, count(*) AS pending_count, min(r.created_at) AS pending_since FROM persistence_requests r
WHERE r.source_id=s.id AND r.source_incarnation=s.incarnation AND r.state IN ('queued','deferred','running','recovering')) p ON true WHERE s.id=$1`, [sourceId]);
const [newest] = row?.pending_min == null ? await engine.executeRaw<{ sequence: number | string | null }>('SELECT max(sequence) AS sequence FROM persistence_requests') : [];
const through = row?.pending_min != null ? Number(row.pending_min) - 1 : newest?.sequence == null ? null : Number(newest.sequence);
return {
source_id: sourceId,
indexed_through: { commit: row?.last_commit ?? null, synced_at: iso(row?.last_sync_at) },
committed_through_request: through,
requests_pending: { count: Number(row?.pending_count ?? 0), oldest_age_seconds: ageSeconds(row?.pending_since, now) },
};
}
/** The source's incarnation and its checkout on this host (the host binding's path, else the source's), or nulls. */
export async function readCheckoutRoot(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string): Promise<{ incarnation: string | null; root: string | null } | undefined> {
const [source] = await engine.executeRaw<{ incarnation: string | null; root: string | null }>(`SELECT s.incarnation::text AS incarnation,
COALESCE((SELECT h.local_path FROM persistence_host_bindings h JOIN persistence_source_bindings b ON b.worktree_id=h.worktree_id
WHERE b.source_id=s.id AND h.host_id::text=$2 LIMIT 1), s.local_path) AS root FROM sources s WHERE s.id=$1`,
[sourceId, (await import('./identity.ts')).existingHostIdFor(engine as BrainEngine)]);
return source;
}
/** The full watermark of one source for `gbrain sources status --json` and MCP `sources_status`. */
export async function readAppliedWatermark(engine: BrainEngine, sourceId: string,
opts: { remote?: boolean; now?: number; behind?: BehindReader; filesBehind?: FilesBehindReader } = {}): Promise<AppliedWatermark> {
const now = opts.now ?? Date.now();
const write = await readWriteWatermark(engine, sourceId, now);
const source = await readCheckoutRoot(engine, sourceId);
const commit = write.indexed_through.commit;
const root = !opts.remote && source?.root && commit ? source.root : null;
const lag = root ? await (opts.behind ?? readGitBehind)(root, commit!).catch(() => null) : null;
const filesBehind = root && lag?.commits ? await (opts.filesBehind ?? readGitFilesBehind)(root, commit!).catch(() => null) : lag ? 0 : null;
const heldFiles = source?.incarnation ? await countGitHolds(engine, sourceId, source.incarnation).catch(() => 0) : 0;
const { readPassReadings } = await import('./sync-pass-readers.ts');
const [pass] = await readPassReadings(engine, { sourceIds: [sourceId], head: false }).catch(() => []);
const deletesWithheld = pass?.row.value.obligations.deletes_withheld ? Number(pass.row.value.telemetry.classes?.deletes_withheld?.count ?? 0) : 0;
const commitsBehind = lag ? { count: lag.commits, oldest_age_seconds: lag.oldest === null ? null : Math.max(0, Math.round(now / 1000 - lag.oldest)) } : null;
const ages = [write.requests_pending.oldest_age_seconds, commitsBehind?.oldest_age_seconds ?? null].filter((age): age is number => age !== null);
let embedded: AppliedWatermark['embedded_through'];
if (await embeddingsDisabled(engine)) embedded = { state: 'embeddings_off', commit: null, before: null, unembedded_pages: 0, held_pages: 0 };
else {
const column = `cc.${quoteIdentifier((await embeddingWriteTarget(engine)).column.name)}`;
const [waiting] = await engine.executeRaw<{ pages: number | string; oldest: unknown; held: number | string }>(`WITH w AS (SELECT DISTINCT p.id, p.updated_at FROM content_chunks cc
JOIN pages p ON p.id=cc.page_id WHERE ${column} IS NULL AND p.source_id=$1 AND p.deleted_at IS NULL AND ${chunkGenerationFilter('p', 'cc')}
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip'))
SELECT count(*) AS pages, min(w.updated_at) AS oldest,
(SELECT count(*) FROM w JOIN page_embedding_budget b ON b.page_id=w.id JOIN pages p ON p.id=w.id WHERE ${budgetRowAppliesFilter('b', 'p')}) AS held FROM w`, [sourceId]);
const pages = Number(waiting?.pages ?? 0);
embedded = pages ? { state: 'partial', commit: null, before: iso(waiting?.oldest), unembedded_pages: pages, held_pages: Number(waiting?.held ?? 0) }
: { state: 'complete', commit, before: null, unembedded_pages: 0, held_pages: 0 };
}
const lease = await readLiveDrainLease(engine, sourceId).catch(() => null);
const holder = lease?.holder ?? null;
const shown = opts.remote ? publicDrainHolder(holder) : holder;
const lastProgress = holder?.progress?.last_progress_at ?? null;
return {
source_id: sourceId,
read_at: new Date(now).toISOString(),
indexed_through: write.indexed_through,
embedded_through: embedded,
committed_through_request: write.committed_through_request,
pending: {
count: write.requests_pending.count + (filesBehind ?? 0) + heldFiles + deletesWithheld,
oldest_age_seconds: ages.length ? Math.max(...ages) : null,
requests: write.requests_pending,
commits_behind: commitsBehind,
files_behind: filesBehind,
held_files: heldFiles,
deletes_withheld: deletesWithheld,
},
writer_heartbeat: {
holder: shown ? { kind: shown.kind, id: shown.id, host: shown.host, pid: shown.pid } : null,
heartbeat_age_seconds: ageSeconds(holder?.last_heartbeat_at, now),
last_progress_at: lastProgress,
progress_age_seconds: ageSeconds(lastProgress, now),
stalled: holderStalled(holder, opts.now ?? lease?.db_now),
},
};
}
/** One status line: `<source>: indexed through <sha>, embedded <state>, N pending (oldest ...), writer ...`. */
export function formatAppliedWatermark(w: AppliedWatermark): string {
const age = (seconds: number | null) => seconds === null ? '' : seconds < 120 ? `${seconds}s` : seconds < 7200 ? `${Math.round(seconds / 60)}m` : `${Math.round(seconds / 3600)}h`;
const embedded = w.embedded_through.state === 'complete' ? 'embedded through it'
: w.embedded_through.state === 'embeddings_off' ? 'embeddings off'
: `${w.embedded_through.unembedded_pages} page(s) wait for vectors${w.embedded_through.held_pages ? ` (${w.embedded_through.held_pages} held)` : ''}`;
const parts = [w.pending.requests.count && `${w.pending.requests.count} request(s)`, w.pending.files_behind && `${w.pending.files_behind} file(s) behind`,
w.pending.held_files && `${w.pending.held_files} held`, w.pending.deletes_withheld && `${w.pending.deletes_withheld} delete(s) withheld`].filter(Boolean);
const pending = parts.length ? `pending: ${parts.join(', ')}${w.pending.oldest_age_seconds !== null ? ` (oldest ${age(w.pending.oldest_age_seconds)})` : ''}` : 'nothing pending';
const holder = w.writer_heartbeat.holder;
const writer = holder ? `; writer ${holder.kind}${holder.pid ? ` PID ${holder.pid}` : ''}${w.writer_heartbeat.stalled ? ' stalled' : ''}${w.writer_heartbeat.progress_age_seconds !== null ? `, last progress ${age(w.writer_heartbeat.progress_age_seconds)} ago` : ''}` : '';
return `${w.source_id}: indexed through ${w.indexed_through.commit?.slice(0, 12) ?? '(never synced)'}, ${embedded}; ${pending}${writer}`;
}