Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #5988: batched Git reads for managed sync hold checks and dry runs: blob ids
* and sizes through chunked `ls-tree`, contents through `cat-file --batch`,
* never one process per file and never an unbounded full-tree read.
* Managed sync's freeze reads its pinned blobs through `readPinnedBlob` /
* `readPinnedContent`: one `ls-tree` per window of upcoming entries and one
* `cat-file --batch` per bounded slice of it, instead of an `ls-tree` and a
* `git show` per entry. GST-2331: every read runs with `HARDENED_GIT_ARGS` and `NO_LAZY_FETCH_ENV`, so no repository
* program runs and a partial clone fetches nothing: an object the repository lacks reads as missing.
*/
import { execFileSync } from 'node:child_process';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { gitChildEnv, literalPathspecs } from '../git-env.ts';
import { HARDENED_GIT_ARGS, noLazyFetchEnv } from '../hardened-git.ts';
import { PAGE_MAX_BYTES } from '../page-size-policy.ts';
/** The sync read bound (`readSyncFile`): bigger files are held without being read. The page ceiling (page-size-policy.ts), which also covers MAX_IMAGE_BYTES. */
export const SYNC_READ_BOUND = PAGE_MAX_BYTES;
export interface TreeBlob { oid: string; size: number }
function git(gitRoot: string, args: string[], input?: string, maxBuffer = 32 * 1024 ** 2, env: NodeJS.ProcessEnv = {}): Buffer {
return execFileSync('git', ['-c', 'core.quotepath=false', ...HARDENED_GIT_ARGS, '-C', gitRoot, ...args],
{ env: gitChildEnv({ ...noLazyFetchEnv(), ...env }), timeout: 60_000, maxBuffer, stdio: ['pipe', 'pipe', 'pipe'], ...(input !== undefined ? { input } : {}) });
}
/** At this many paths `readTreeBlobs` lists the whole tree once instead of path chunks (#6712: the prescreen of a tree walk). */
const WHOLE_TREE_AT = 1000;
/** Blob id and size of Git paths at a commit, in chunked `ls-tree` calls (never one process per file), or one recursive listing for a large set. */
export function readTreeBlobs(gitRoot: string, target: string, gitPaths: string[]): Map<string, TreeBlob> {
const out = new Map<string, TreeBlob>();
const whole = gitPaths.length >= WHOLE_TREE_AT, wanted = whole ? new Set(gitPaths) : null;
for (let i = 0; i < gitPaths.length; i += 500) {
const rows = (whole ? git(gitRoot, ['ls-tree', '-r', '-l', '-z', target], undefined, 512 * 1024 ** 2)
: git(gitRoot, ['ls-tree', '-l', '-z', target, '--', ...literalPathspecs(gitPaths.slice(i, i + 500))])).toString('utf8').split('\0');
for (const row of rows) {
const tab = row.indexOf('\t');
if (tab < 0 || wanted && !wanted.has(row.slice(tab + 1))) continue;
const [, kind, oid, size] = row.slice(0, tab).split(/\s+/);
if (kind === 'blob') out.set(row.slice(tab + 1), { oid: oid!, size: Number(size) });
}
if (whole) break;
}
return out;
}
/** Decoded (UTF-8) blob contents by id, read through `git cat-file --batch` in bounded chunks. */
export function readBlobContents(gitRoot: string, blobs: TreeBlob[]): Map<string, string> {
return new Map([...readBlobBytes(gitRoot, blobs)].map(([oid, bytes]) => [oid, bytes.toString('utf8')]));
}
/** #6529: the raw bytes of blobs by id (an image import keeps its bytes), read through `git cat-file --batch` in bounded chunks. */
export function readBlobBytes(gitRoot: string, blobs: TreeBlob[]): Map<string, Buffer> {
const out = new Map<string, Buffer>();
const pending = [...new Map(blobs.filter(blob => blob.size <= SYNC_READ_BOUND).map(blob => [blob.oid, blob])).values()];
while (pending.length) {
const chunk: TreeBlob[] = [];
let bytes = 0;
while (pending.length && (chunk.length === 0 || bytes + pending[0]!.size <= 24 * 1024 ** 2)) { bytes += pending[0]!.size; chunk.push(pending.shift()!); }
const output = git(gitRoot, ['cat-file', '--batch'], chunk.map(blob => blob.oid).join('\n') + '\n', bytes + chunk.length * 128 + 1024);
let offset = 0;
for (const blob of chunk) {
const newline = output.indexOf(10, offset);
const [oid, kind, size] = output.subarray(offset, newline).toString('utf8').split(' ');
if (kind === 'missing' || size === undefined) { offset = newline + 1; continue; }
const start = newline + 1, end = start + Number(size);
out.set(oid!, Buffer.from(output.subarray(start, end)));
offset = end + 1;
}
}
return out;
}
let checkAttrSource: boolean | undefined;
export { __setNoLazyFetchForTests } from '../hardened-git.ts';
/** Test seam: `false` takes the path of a git older than 2.40; returns the restore. */
export function __setCheckAttrSourceForTests(value: boolean | undefined): () => void {
const previous = checkAttrSource;
checkAttrSource = value;
return () => { checkAttrSource = previous; };
}
/** GST-2312 (#9024): a `.gitattributes` of the commit that this repository lacks (a partial clone that never fetched it). */
export interface MissingAttributes { attributes: string; oid: string }
/**
* GST-2312 (#9024): the Git filter driver each path's `filter` attribute names at commit `target`, null where none does,
* whether or not this checkout's Git config defines the driver (the commit may have been cleaned on another machine). A
* committed blob a filter owns holds what the filter's clean program stored (git-crypt ciphertext, for one), not the text
* of the checkout, and sync never runs a repository's filters. Git 2.40 and newer read the attributes with
* `check-attr --source=<target>`; an older git reads `target` into a temporary index (`read-tree`) and asks
* `check-attr --cached` there. Like every read here, each call fetches nothing in a partial clone (GST-2331). Git then reads a `.gitattributes` the repository lacks as empty, so
* the ones that apply are checked first (a git older than 2.45 fails the batch on a promisor object instead, so each is then checked alone), and a path
* under a missing one maps to it instead of to a driver and is not passed to `check-attr`.
*/
export function readFilterDrivers(gitRoot: string, target: string, gitPaths: string[]): Map<string, string | null | MissingAttributes> {
const out = new Map<string, string | null | MissingAttributes>();
if (gitPaths.length === 0) return out;
const read = (args: string[], input?: string, env: NodeJS.ProcessEnv = {}) => git(gitRoot, args, input, undefined, env);
if (checkAttrSource === undefined) {
const [major, minor] = (/git version (\d+)\.(\d+)/.exec(read(['version']).toString('utf8')) ?? ['', '0', '0']).slice(1).map(Number);
checkAttrSource = major! > 2 || major === 2 && minor! >= 40;
}
const dirsOf = (path: string) => path.split('/').slice(0, -1).map((_, at, parts) => parts.slice(0, at + 1).join('/')).concat('');
const candidates = [...new Set(gitPaths.flatMap(dirsOf))].map(dir => dir ? `${dir}/.gitattributes` : '.gitattributes');
const present: MissingAttributes[] = [];
for (let i = 0; i < candidates.length; i += 1000) {
for (const record of read(['ls-tree', '-z', '--full-tree', target, '--', ...candidates.slice(i, i + 1000)]).toString('utf8').split('\0')) {
const match = /^\d+ blob ([0-9a-f]+)\t(.*)$/s.exec(record);
if (match) present.push({ attributes: match[2]!, oid: match[1]! });
}
}
const missing = new Map<string, MissingAttributes>();
let checked: string[] = [];
try {
if (present.length) checked = read(['cat-file', '--batch-check'], present.map(file => file.oid).join('\n') + '\n').toString('utf8').split('\n');
} catch (error) {
if (!String((error as { stderr?: unknown }).stderr ?? '').includes('from promisor remote')) throw error;
checked = present.map(file => {
try { read(['cat-file', '-e', file.oid]); return ''; } catch { return `${file.oid} missing`; }
});
}
present.forEach((file, at) => { if (checked[at]?.endsWith(' missing')) missing.set(file.attributes.slice(0, -'.gitattributes'.length).replace(/\/$/, ''), file); });
const screened = gitPaths.filter(path => {
const lost = dirsOf(path).map(dir => missing.get(dir)).find(Boolean);
if (lost) out.set(path, lost);
return !lost;
});
const scratch = checkAttrSource ? null : mkdtempSync(join(tmpdir(), 'gbrain-attributes-'));
const env = scratch ? { GIT_INDEX_FILE: join(scratch, 'index') } : {};
try {
if (scratch) read(['read-tree', target], undefined, env);
for (let i = 0; i < screened.length; i += 2000) {
const fields = read(['check-attr', ...(scratch ? ['--cached'] : [`--source=${target}`]), '-z', '--stdin', 'filter'],
screened.slice(i, i + 2000).join('\0') + '\0', env).toString('utf8').split('\0');
for (let at = 0; at + 2 < fields.length; at += 3) out.set(fields[at]!, ['unspecified', 'unset', 'set'].includes(fields[at + 2]!) ? null : fields[at + 2]!);
}
} finally {
if (scratch) rmSync(scratch, { recursive: true, force: true });
}
return out;
}
/** Entries one pinned `ls-tree` covers, and the bytes one window's `cat-file --batch` loads at most. */
export const PINNED_WINDOW = 256;
const PINNED_WINDOW_BYTES = 8 * 1024 ** 2;
/** Loaded byte slices one window keeps: a follower batch reads at most four neighbouring entries at once. */
const PINNED_SLICES_KEPT = 4;
const COMMIT_ID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
/**
* The pinned tree window of the run reading now. A full commit id names
* immutable trees and blobs, so an answer read for a later entry of the window
* is the answer that entry's own read would give; a different root or target
* replaces the window.
*/
let pinned: PinnedWindow | null = null;
interface PinnedWindow { gitRoot: string; target: string; paths: string[]; blobs: Map<string, TreeBlob | null>; slices: number[] | null; loaded: Array<{ slice: number; contents: Map<string, Buffer> }> }
let pinnedWindowSize = PINNED_WINDOW;
/** Test seam: replace the pinned window size (0 reads every entry alone, as before windows; null restores it); returns the restore function. */
export function __setPinnedWindowForTests(size: number | null): () => void {
const previous = pinnedWindowSize;
pinnedWindowSize = size ?? PINNED_WINDOW;
pinned = null;
return () => { pinnedWindowSize = previous; pinned = null; };
}
/**
* The aligned block of manifest indexes whose committed imports one pinned window lists. Every entry of a block asks
* for the same window, so concurrent follower freezes, which reach their pinned read in any order, never list it twice.
*/
export function pinnedWindowBlock(index: number): { start: number; end: number } {
const start = pinnedWindowSize ? index - index % pinnedWindowSize : index;
return { start, end: start + (pinnedWindowSize || PINNED_WINDOW) };
}
/**
* #6712 PR 1: installs a window another reader already listed (the discovery prescreen's `ls-tree` of the frozen
* manifest's first block), so the run's first pinned read does not list the same paths again.
*/
export function seedPinnedWindow(gitRoot: string, target: string, paths: string[], found: ReadonlyMap<string, TreeBlob>): void {
if (!pinnedWindowSize || !COMMIT_ID.test(target) || !paths.length) return;
const kept = paths.slice(0, pinnedWindowSize);
pinned = { gitRoot, target, paths: kept, blobs: new Map(kept.map(path => [path, found.get(path) ?? null])), slices: null, loaded: [] };
}
function pinnedWindow(gitRoot: string, target: string, gitPath: string, upcoming: () => string[]) {
if (pinned?.gitRoot === gitRoot && pinned.target === target && pinned.blobs.has(gitPath)) return pinned;
const paths = [gitPath, ...new Set(upcoming().filter(path => path !== gitPath))].slice(0, pinnedWindowSize);
let found: Map<string, TreeBlob>;
// A window that cannot be listed (one path git rejects) falls back to this entry alone, which reports its own error.
try { found = readTreeBlobs(gitRoot, target, paths); } catch { return null; }
pinned = { gitRoot, target, paths, blobs: new Map(paths.map(path => [path, found.get(path) ?? null])), slices: null, loaded: [] };
return pinned;
}
/**
* Where the window's byte slices start: cut in window order by `PINNED_WINDOW_BYTES` alone, so which entry asks first
* (concurrent follower freezes finish in any order) never moves a boundary or adds a `cat-file`.
*/
function windowSlices(window: PinnedWindow): number[] {
const starts: number[] = [];
let bytes = 0;
window.paths.forEach((path, index) => {
const blob = window.blobs.get(path);
if (!blob || blob.size > SYNC_READ_BOUND) return;
if (!starts.length || bytes + blob.size > PINNED_WINDOW_BYTES) { starts.push(index); bytes = 0; }
bytes += blob.size;
});
return starts;
}
/**
* The blob of `gitPath` at the pinned `target`, or null when the commit has no
* blob there: the answer of `readTreeBlobs` for that one path. `upcoming` lists
* the Git paths of the entry's aligned block (`pinnedWindowBlock`); a path
* outside the current window lists itself and them in one `ls-tree`.
*/
export function readPinnedBlob(gitRoot: string, target: string, gitPath: string, upcoming: () => string[]): TreeBlob | null {
const window = pinnedWindowSize && COMMIT_ID.test(target) ? pinnedWindow(gitRoot, target, gitPath, upcoming) : null;
return window ? window.blobs.get(gitPath)! : readTreeBlobs(gitRoot, target, [gitPath]).get(gitPath) ?? null;
}
/**
* The decoded content of the blob at `gitPath` in the pinned `target`, or null
* when the window cannot answer it (no blob there, over the sync read bound, or
* unreadable); the caller then reads it as before. A miss loads the byte slice
* holding this blob in one `cat-file --batch`; the last `PINNED_SLICES_KEPT`
* slices stay loaded, so reads in any order cost one process per slice.
*/
export function readPinnedContent(gitRoot: string, target: string, gitPath: string, upcoming: () => string[]): string | null {
return readPinnedBytes(gitRoot, target, gitPath, upcoming)?.toString('utf8') ?? null;
}
/** #6529: `readPinnedContent` before decoding: the raw bytes of the pinned blob (an image import), or null when the window cannot answer. */
export function readPinnedBytes(gitRoot: string, target: string, gitPath: string, upcoming: () => string[]): Buffer | null {
if (!pinnedWindowSize || !COMMIT_ID.test(target)) return null;
const window = pinnedWindow(gitRoot, target, gitPath, upcoming);
const blob = window?.blobs.get(gitPath);
if (!window || !blob || blob.size > SYNC_READ_BOUND) return null;
const starts = window.slices ??= windowSlices(window);
const at = window.paths.indexOf(gitPath);
const slice = starts.findLastIndex(start => start <= at);
let loaded = window.loaded.find(entry => entry.slice === slice);
if (!loaded) {
const blobs = window.paths.slice(starts[slice], starts[slice + 1] ?? window.paths.length).map(path => window.blobs.get(path))
.filter((next): next is TreeBlob => !!next && next.size <= SYNC_READ_BOUND);
try { loaded = { slice, contents: readBlobBytes(gitRoot, blobs) }; } catch { return null; }
window.loaded = [...window.loaded, loaded].slice(-PINNED_SLICES_KEPT);
}
return loaded.contents.get(blob.oid) ?? null;
}