Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { randomUUID } from 'node:crypto';
import { existsSync, lstatSync } from 'node:fs';
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import { isImageFilePath, type ImportResult } from '../import-file.ts';
import { loadConfig } from '../config.ts';
import { opError, OperationError, type OperationContext } from '../ops/contract.ts';
import { currentSubmissionAuthority } from '../minions/submission-authority.ts';
import { currentVerifiedLocalWriter, hostIdFor } from './identity.ts';
import { getWorktreeBinding } from './ownership.ts';
import { SCREENING_SHARED_READS, batchSharedReads, emitFenceNotice, initializeLocalPersistence, pendingAwareResponse, preparePageAdmission, requestPrincipalForContext, withBatchAdmission,
withLiveSearchable } from './page-mutations.ts';
import { admitBatch } from './page-batch.ts';
import { writeOutcomeUnknown } from './admission-retry.ts';
import { assertPersistenceAccepting, waitForWrites } from './service.ts';
import type { WriteAdmission } from './journal.ts';
import type { WriteRequest } from './model.ts';
import { digest, sha256 } from './digest.ts';
import { assertImportPaths, managedImportContent, managedStreamedIdentity, prepareManagedImportMutation, readImportBytes, type ImportPack, type ManagedImportBodyRef,
type ManagedImportIntent } from './import-prepare.ts';
import { readPageBodyPolicy } from '../page-body/limits.ts';
import { submissionAuthority } from './authority.ts';
import { inspectUnchanged, screeningRequest } from './noop-kernel.ts';
import { withScreeningPaths } from './screening-paths.ts';
import { readImportSnapshots } from './import-reads.ts';
import { pageSnapshotKey } from '../page-snapshot-batch.ts';
import type { WorktreeBinding } from './ownership.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import { readFix, trustedCliRequired } from '../ops/op-fix.ts';
import { writeThroughConfigured } from './switches.ts';
/** One file of a managed import batch: its path and the source-relative path its slug is derived from. */
export interface ManagedImportFile { filePath: string; sourcePath: string }
/** A managed import admits at most this many files in one transaction; their publication groups hold PAGE_BATCH_GROUP_MAX pages each. */
export const IMPORT_BATCH_MAX_PAGES = 32;
export const IMPORT_BATCH_MAX_BYTES = 8 * 1024 * 1024;
/**
* The files of `queue` from `start` that one `runImport` batch takes: at least one, at most the page and byte caps;
* exactly one when the import is not a managed one (`batched` false). `runImport` accounts each file of a batch, in
* order, through its per-file bookkeeping.
*/
export function nextImportBatch(queue: readonly string[], start: number, batched = true): string[] {
if (!batched) return [queue[start]!];
const batch: string[] = [];
let bytes = 0;
for (let i = start; i < queue.length && batch.length < IMPORT_BATCH_MAX_PAGES; i++) {
const size = lstatSync(queue[i]!, { throwIfNoEntry: false })?.size ?? 0;
if (batch.length && bytes + size > IMPORT_BATCH_MAX_BYTES) break;
batch.push(queue[i]!);
bytes += size;
}
return batch;
}
/**
* One `runImport` batch: the result of every file of `batch` in its order, undefined for a held path (the caller
* accounts it as held). A batch that fails as a whole settles each of its files with that failure.
*/
export async function settleManagedImportBatch(engine: BrainEngine, batch: readonly string[], sourcePathOf: (file: string) => string,
held: (sourcePath: string) => boolean, opts: Parameters<typeof importManagedFiles>[2]): Promise<Array<PromiseSettledResult<ImportResult> | undefined>> {
const open = batch.filter(file => !held(sourcePathOf(file)));
const t0 = Date.now();
const settled: PromiseSettledResult<ImportResult>[] = !open.length ? [] : await importManagedFiles(engine, open.map(filePath => ({ filePath, sourcePath: sourcePathOf(filePath) })), opts)
.catch(reason => open.map(() => ({ status: 'rejected' as const, reason })));
if (Date.now() - t0 > 5000 * open.length) console.error(`[gbrain phase] import.process_batch slow ${Date.now() - t0}ms files=${open.length} first=${sourcePathOf(open[0]!)}`);
return batch.map(file => settled[open.indexOf(file)]);
}
const IMPORT_OP = 'managed-file-import';
/** How long an import waits for one file's publication, as a single import always has. */
const IMPORT_WAIT_MS = 30_000;
type PendingImport = ManagedImportIntent & { request_id: string; source_id: string };
/**
* GBRA-69: the durable pre-admission checkpoint holds the admission's expectations and request id,
* not the file's content: its key binds the input hash, so a retry restores the content from the
* bytes it just read (and refuses when they no longer hash to the stored input). Checkpoints older
* binaries wrote still carry the content; their row is matched as stored.
*/
type StoredImport = Omit<PendingImport, 'content'> & { content?: string };
/**
* #7085: with `sync.write_through` off a page write records no worktree, and a managed import publishes only through
* the source's canonical owner, so every file would be admitted and then refused. The import refuses once, before any admission.
*/
export async function managedImportWriteThroughRefusal(engine: BrainEngine, sourceId: string): Promise<OperationError | null> {
if (writeThroughConfigured(await engine.getConfig('sync.write_through'))) return null;
return opError('owner_unavailable', 'Managed import publishes through the canonical owner, and sync.write_through is off on this brain.',
`Nothing was imported and no write was admitted. Copy the files into source ${sourceId}'s checkout, commit them, then run gbrain sync --source ${sourceId}.`,
{ detail: 'disabled_by_config', retryable: false, fix: { argv: ['gbrain', 'sync', '--source', sourceId], consent: [], actor: 'agent',
why: `Imports the files committed in source ${sourceId}'s checkout; run it after committing them.`, requires_exclusive: false } });
}
export async function importManagedFile(engine: BrainEngine, filePath: string, sourcePath: string,
opts: { sourceId?: string; noEmbed?: boolean; activePack?: ImportPack; signal?: AbortSignal; slugRoot?: string } = {}): Promise<ImportResult> {
const [settled] = await importManagedFiles(engine, [{ filePath, sourcePath }], opts);
if (settled!.status === 'rejected') throw settled!.reason;
return settled!.value;
}
/**
* Imports files through the source's canonical owner. Every file keeps its own
* durable intent (op_checkpoints), request, receipt and result; a batch admits
* its new requests in one transaction, marked with one `import_batch` id so the
* owner publishes them in groups (journal.ts publicationGroupKey). A file's
* refusal settles that file only. A slug a batch already holds ends the batch
* there; that file and the rest import after it, as they would one by one.
*/
export async function importManagedFiles(engine: BrainEngine, files: readonly ManagedImportFile[],
opts: { sourceId?: string; noEmbed?: boolean; activePack?: ImportPack; signal?: AbortSignal; slugRoot?: string } = {}): Promise<PromiseSettledResult<ImportResult>[]> {
const caller = currentSubmissionAuthority();
if (caller && caller.kind !== 'application' || currentVerifiedLocalWriter()?.remote) {
throw trustedCliRequired('Managed filesystem import requires the trusted local CLI.');
}
opts.signal?.throwIfAborted();
const settled: PromiseSettledResult<ImportResult>[] = new Array(files.length);
const fail = (i: number, reason: unknown) => { settled[i] = { status: 'rejected', reason }; };
const sourceId = opts.sourceId ?? 'default';
const binding = await getWorktreeBinding(engine, sourceId);
const unowned = !binding?.local_path || binding.owner_host_id !== hostIdFor(engine) || binding.state !== 'active'
? opError('owner_unavailable', 'Managed import must run on the active canonical owner for the selected source.',
`This host is not the active owner of source ${sourceId}, so nothing was imported. Run the import on the owner host that writer status names, or wait until its owner is active.`,
{ fix: readFix(`Shows source ${sourceId}'s owner host and state, read-only.`, { argv: ['gbrain', 'sources', 'writer', 'status', '--source', sourceId, '--json'] }) }) : null;
const ctx = { engine, remote: false, sourceId, config: loadConfig() ?? { engine: engine.kind } } as OperationContext;
const reads = batchSharedReads(engine);
const bodyPolicy = await readPageBodyPolicy(engine);
const planned: Array<{ i: number; slug: string; content?: string; bodyRef?: ManagedImportBodyRef; sourcePath: string; path: string; inputPath: string; inputHash: string; target: string }> = [];
const slugs = new Set<string>();
let rest = files.length;
await withScreeningPaths(async () => { for (let i = 0; i < files.length; i++) {
let { filePath, sourcePath } = files[i]!;
try {
if (isImageFilePath(sourcePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL !== 'true') {
throw opError('invalid_params', 'Image import requires GBRAIN_EMBEDDING_MULTIMODAL=true.',
`${sourcePath} is an image and multimodal embeddings are off, so it was not imported. Import only text files, or ask the user whether to enable GBRAIN_EMBEDDING_MULTIMODAL=true for the gbrain process (image embeddings can cost money).`);
}
if (unowned) throw unowned;
const root = join(binding!.local_path!, binding!.relative_path);
const inputPath = resolve(filePath);
const canonicalRelative = relative(root, inputPath);
const canonicalInput = canonicalRelative && !isAbsolute(canonicalRelative) && canonicalRelative !== '..' && !canonicalRelative.startsWith(`..${sep}`);
if (canonicalInput && !opts.slugRoot) sourcePath = canonicalRelative;
const path = canonicalInput ? canonicalRelative : sourcePath;
const target = resolve(root, path);
await assertImportPaths(reads, sourceId, root, inputPath, target);
const streamed = await managedStreamedIdentity(sourcePath, inputPath, target, bodyPolicy, sourceId);
const bytes = streamed ? null : readImportBytes(inputPath);
const { slug, content } = streamed ? { slug: streamed.slug, content: undefined } : managedImportContent(sourcePath, bytes!, opts.activePack);
if (slugs.has(slug)) { rest = i; break; }
slugs.add(slug);
planned.push({ i, slug, content, ...(streamed ? { bodyRef: streamed.bodyRef } : {}), sourcePath, path, inputPath,
inputHash: streamed ? streamed.bodyRef.sha256 : sha256(bytes!), target });
} catch (error) { fail(i, error); }
} });
if (planned.length) await importPlanned(ctx, binding!, planned, opts, fail, (i, value) => { settled[i] = { status: 'fulfilled', value }; });
if (rest < files.length) settled.splice(rest, files.length - rest, ...await importManagedFiles(engine, files.slice(rest), opts));
return settled;
}
async function importPlanned(ctx: OperationContext, binding: WorktreeBinding,
planned: Array<{ i: number; slug: string; content?: string; bodyRef?: ManagedImportBodyRef; sourcePath: string; path: string; inputPath: string; inputHash: string; target: string }>,
opts: { noEmbed?: boolean; activePack?: ImportPack },
fail: (i: number, reason: unknown) => void, done: (i: number, value: ImportResult) => void): Promise<void> {
const { engine } = ctx;
const sourceId = binding.source_id;
let principal: Awaited<ReturnType<typeof requestPrincipalForContext>>;
try {
await initializeLocalPersistence(ctx);
principal = await requestPrincipalForContext(ctx);
} catch (error) { for (const entry of planned) fail(entry.i, error); return; }
const members = planned.map(entry => ({ ...entry, key: digest({ principal, incarnation: binding.source_incarnation, inputPath: entry.inputPath, sourcePath: entry.sourcePath,
path: entry.path, inputHash: entry.inputHash, noEmbed: !!opts.noEmbed, activePack: opts.activePack ?? null }),
params: undefined as PendingImport | undefined, stored: undefined as StoredImport | undefined }));
const readPending = async (keys: string[]) => new Map((await engine.executeRaw<{ fingerprint: string; completed_keys: [StoredImport] }>(
'SELECT fingerprint,completed_keys FROM op_checkpoints WHERE op=$1 AND fingerprint=ANY($2::text[])', [IMPORT_OP, keys])).map(row => [row.fingerprint, row.completed_keys[0]]));
const resume = (member: (typeof members)[number], stored: StoredImport) => {
// The key binds the input hash and slug, so a mismatch is a checkpoint this binary cannot pair with the bytes.
if (stored.inputHash !== member.inputHash || stored.slug !== member.slug) throw opError('internal_error', 'A pending import checkpoint does not match the file it is keyed by.',
`The checkpointed import of ${member.sourcePath} (request ${stored.request_id}) in source ${sourceId} records a different input than the bytes just read under the same key, so nothing was resubmitted. Report it with gbrain doctor --json.`);
member.stored = stored;
member.params = { ...stored, ...(member.content !== undefined ? { content: member.content } : {}) } as PendingImport;
};
const open = new Set(members);
const settle = (member: (typeof members)[number], value: ImportResult) => { open.delete(member); done(member.i, value); };
const refuse = (member: (typeof members)[number], reason: unknown) => { open.delete(member); fail(member.i, reason); };
const batch = randomUUID();
// The no-op screen only decides whether a file is skipped; a file it does not
// skip is admitted and rechecked under lock, so the batch answers its shared reads once.
const screening: OperationContext = { ...ctx, engine: batchSharedReads(engine, SCREENING_SHARED_READS) };
try {
const pending = await readPending(members.map(member => member.key));
const fresh: typeof members = [];
// GBRA-75 wave 10: the screen reads its new members' pages in one batch; a lone member or a failed batch keeps each member's own read.
const unscreened = members.filter(member => !pending.has(member.key)).map(member => ({ slug: member.slug, sourceId }));
const screened = unscreened.length > 1 ? await readImportSnapshots(engine, unscreened).catch(() => null) : null;
await withScreeningPaths(async () => { for (const member of members) {
try {
const stored = pending.get(member.key);
if (stored) { resume(member, stored); continue; }
const snapshot = screened ? screened.get(pageSnapshotKey(sourceId, member.slug)) ?? null : await engine.readPageSnapshot(member.slug, { sourceId, includeDeleted: true });
// The input is the canonical file itself (the common case): its hash was taken from the bytes just read.
const targetHash = member.target === member.inputPath ? member.inputHash : existsSync(member.target) ? sha256(readImportBytes(member.target)) : null;
const intent: ManagedImportIntent = { kind: 'managed_file_import', slug: member.slug, ...(member.bodyRef ? { body_ref: member.bodyRef } : { content: member.content }),
sourcePath: member.sourcePath, path: member.path,
inputPath: member.inputPath, inputHash: member.inputHash, targetHash,
ownerEpoch: String(binding.owner_epoch), ...(snapshot ? { expected_revision: snapshot.revision } : {}),
noEmbed: !!opts.noEmbed, ...(opts.activePack ? { activePack: opts.activePack } : {}) };
// #5470: an import whose publication would change nothing takes no admission.
if (await unchangedManagedImport(screening, binding, intent, snapshot)) { settle(member, { slug: member.slug, status: 'skipped', chunks: 0 }); continue; }
member.params = { ...intent, ...(members.length > 1 ? { import_batch: batch } : {}), request_id: randomUUID(), source_id: sourceId };
fresh.push(member);
} catch (error) { refuse(member, error); }
} });
if (fresh.length) {
await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys)
SELECT $1,k,c::jsonb FROM unnest($2::text[],$3::text[]) AS u(k,c) ON CONFLICT DO NOTHING`,
[IMPORT_OP, fresh.map(member => member.key), fresh.map(member => { const { content: _content, ...stored } = member.params!; return JSON.stringify([stored]); })]);
const stored = await readPending(fresh.map(member => member.key));
for (const member of fresh) { try { resume(member, stored.get(member.key)!); } catch (error) { refuse(member, error); } }
}
} catch (error) { for (const member of [...open]) refuse(member, error); return; }
const submitted = members.filter(member => open.has(member));
if (!submitted.length) return;
const outcomes = await submitManagedImports(ctx, batch, submitted.map(member => member.params!));
const cleared: typeof submitted = [];
submitted.forEach((member, position) => {
const outcome = outcomes[position]!;
if (outcome.status === 'fulfilled') {
cleared.push(member);
const value = outcome.value;
done(member.i, { slug: String(value.slug), status: value.status as ImportResult['status'], chunks: Number(value.chunks ?? 0),
...(typeof value.error === 'string' ? { error: value.error } : {}),
...(value.searchable !== undefined ? { searchable: value.searchable as ImportResult['searchable'] } : {}),
...(value.searchable === 'pending' ? { projection: 'pending' as const, request_id: String(value.request_id),
searchable_retry_after_ms: Number(value.searchable_retry_after_ms), searchable_fix: value.searchable_fix as ImportResult['searchable_fix'] } : {}),
...(value.type_warning ? { type_warning: value.type_warning as ImportResult['type_warning'] } : {}) });
return;
}
const error = outcome.reason;
if (error instanceof OperationError && error.writeRequest && ['failed', 'conflict', 'cancelled'].includes(error.writeRequest.state)) cleared.push(member);
fail(member.i, error);
});
if (!cleared.length) return;
try {
await engine.executeRaw(`DELETE FROM op_checkpoints o USING unnest($2::text[],$3::text[]) AS d(k,c)
WHERE o.op=$1 AND o.fingerprint=d.k AND o.completed_keys=d.c::jsonb`,
[IMPORT_OP, cleared.map(member => member.key), cleared.map(member => JSON.stringify([member.stored]))]);
} catch (error) { for (const member of cleared) fail(member.i, error); }
}
/**
* The batch's owner-file `put_page` intents prepared under one writer
* verification, the new ones admitted in one transaction (page-batch.ts
* admitBatch) and awaited together. Each settles as submitPageMutation
* would settle it alone; a request still pending after its share of the
* wait settles as `write_pending`.
*/
async function submitManagedImports(ctx: OperationContext, batch: string, params: readonly PendingImport[]): Promise<PromiseSettledResult<Record<string, unknown>>[]> {
const settled: PromiseSettledResult<Record<string, unknown>>[] = new Array(params.length);
const rows: Array<{ index: number; row: WriteRequest; admitted: boolean; typeWarning?: unknown; slugAdvisory?: unknown }> = [];
try {
assertPersistenceAccepting(ctx.engine);
await withBatchAdmission(ctx, async (own, shared) => {
const admissions: Array<{ index: number; admission: WriteAdmission; typeWarning: unknown; slugAdvisory: unknown }> = [];
for (let index = 0; index < params.length; index++) {
try {
const prepared = await preparePageAdmission(index === 0 ? own : shared, { operation: 'put_page', params: params[index]!, managedFileImport: true });
if (prepared.prior) rows.push({ index, row: prepared.prior, admitted: false });
else admissions.push({ index, admission: prepared.admission, typeWarning: prepared.typeWarning, slugAdvisory: prepared.slugAdvisory });
} catch (reason) { settled[index] = { status: 'rejected', reason }; }
}
if (!admissions.length) return;
try {
const admitted = await admitBatch(ctx, batch, admissions.map(entry => entry.admission));
admissions.forEach((entry, position) => rows.push({ index: entry.index, row: admitted[position]!, admitted: true, typeWarning: entry.typeWarning, slugAdvisory: entry.slugAdvisory }));
} catch (reason) {
// #6355: the batch's outcome is unknown for every file; each file names its own request id to read (and replay).
const unknown = reason instanceof OperationError && reason.code === 'write_outcome_unknown';
for (const entry of admissions) settled[entry.index] = { status: 'rejected',
reason: unknown ? writeOutcomeUnknown(entry.admission.requestId!, new Error(String(reason.detail ?? reason.message))) : reason };
}
});
} catch (reason) {
for (let index = 0; index < params.length; index++) settled[index] ??= { status: 'rejected', reason };
return settled;
}
rows.sort((a, b) => a.index - b.index);
const done = await waitForWrites(ctx.engine, rows.map(entry => entry.row), ctx.config, IMPORT_WAIT_MS * rows.length);
await Promise.all(rows.map(async (entry, position) => {
try {
// #7106: a new segmented page over 4 MiB replies at commit with `searchable: 'pending'`, read live here.
const response = await withLiveSearchable(ctx, pendingAwareResponse(ctx, done[position]!));
if (entry.admitted) emitFenceNotice(ctx, response, done[position]!.slug);
settled[entry.index] = { status: 'fulfilled', value: { ...response, ...(entry.typeWarning ? { type_warning: entry.typeWarning } : {}),
...(entry.slugAdvisory ? { slug_advisory: entry.slugAdvisory } : {}) } };
} catch (reason) { settled[entry.index] = { status: 'rejected', reason }; }
}));
return settled;
}
/**
* Runs the managed-import preparer on an unadmitted request; true only when the no-op kernel finds nothing to publish.
* A file admitted by reference (`body_ref`) is not screened: its screen would stream and stage the whole body, so its
* publication decides the no-op.
*/
async function unchangedManagedImport(ctx: OperationContext, binding: WorktreeBinding, intent: ManagedImportIntent, snapshot: PageSnapshot | null): Promise<boolean> {
if (!snapshot || intent.body_ref) return false;
// #6529: an image's preparation is provider work (OCR, the multimodal vector) unless its index is already complete, so the
// screen would pay for every incomplete image and the owner would pay again. The owner decides the hash no-op (one importer call).
if (isImageFilePath(intent.sourcePath)) return false;
try {
const authority = await submissionAuthority(ctx, 'put_page', binding.source_id, binding.source_incarnation, intent.slug);
const row = screeningRequest({ source_id: binding.source_id, source_incarnation: binding.source_incarnation, slug: intent.slug,
page_id: snapshot.page.id, worktree_id: binding.worktree_id, authority, intent, operation: 'put_page' });
const prepared = await prepareManagedImportMutation(ctx.engine, row, ctx.config, { snapshot });
if ((await inspectUnchanged(ctx.engine, { prepared, snapshot, sourcePath: intent.sourcePath, databaseOnly: false })).admitReason) return false;
await prepared.validate?.(ctx.engine);
return true;
} catch {
return false;
}
}