Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* sweep-shared — the per-run dependency bag and the page write/delete
* primitives every Google sweep (contacts, calendar, Gmail) shares. Peeled
* from google-source.ts so the sweeps can live in their own modules.
*/
import { closeSync, existsSync, fchmodSync, lstatSync, openSync, readFileSync, renameSync, rmSync, unlinkSync, writeSync } from 'node:fs';
import { dirname, join } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts } from '../../commands/sync.ts';
import type { CredentialEntry } from '../creds/vault.ts';
import type { ManagedConnectorSync } from '../persistence/connector-sync.ts';
import { connectorRender } from '../connectors/connector-text.ts';
import { isWriteTargetContained } from '../path-confine.ts';
import { mkdirPrivate } from '../atomic-write.ts';
import { replaceableFile, type StoredBody } from '../facts/file-body-guard.ts';
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END, replaceOrInsertFactsFence } from '../facts-fence.ts';
import { carryStoredFences, hasFence } from '../connectors/fence-carry.ts';
import { assertInlineBody } from '../page-body/page-shape.ts';
import { OperationError } from '../ops/contract.ts';
import { slugifyPath } from '../sync.ts';
import { isPrivatePage } from '../search/private-visibility.ts';
import { withPrivateVisibility } from './google-render.ts';
import { isWriteThroughDisabled } from '../write-through.ts';
import { isSourceDbOnlySlug } from '../persistence/source-storage.ts';
import { intentCarriesPurgedContent } from '../persistence/purged-intent.ts';
import type { GoogleSourceConfig, GoogleSourceState } from './types.ts';
import type { LoopsExclusionPolicy } from './loops-exclusion.ts';
/** The "my addresses" identity set: account + Gmail sendAs aliases. */
export function myAddressSet(entry: CredentialEntry): Set<string> {
const out = new Set<string>();
if (entry.meta.account) out.add(entry.meta.account.toLowerCase());
for (const a of entry.meta.sendas_aliases ?? []) out.add(a.toLowerCase());
return out;
}
export interface GoogleSyncSummary {
/** 'up_to_date'/'first_sync' are computed on the SyncResult, never here. */
status: 'synced' | 'partial';
added: number;
modified: number;
deleted: number;
chunksCreated: number;
embedded: number;
pagesAffected: string[];
threadsSeen: number;
attachmentInspection: Record<string, number>;
/**
* Why each in-window thread was or was not sent to the extractor, keyed by
* the machine reason from loopExtractionEligibility. Counts only — no
* addresses, subjects or body text — so a sweep can be audited for
* over-filtering without leaking mail content into logs.
*/
extractEligibility: Record<string, number>;
/** #5012: reply-owed loops the detector withheld this sweep, by reason (counts only). */
loopsWithheld: Record<string, number>;
failedFiles: number;
}
export interface GoogleSyncDeps {
managed: ManagedConnectorSync | null;
engine: BrainEngine;
sourceId: string;
cfg: GoogleSourceConfig;
opts: SyncOpts;
entry: CredentialEntry;
log: (msg: string) => void;
/**
* #5349: one forward-progress note per imported item. The progress-aware
* sync deadline extends only on these (heartbeats are not progress), so a
* long first contacts or calendar sweep that is still landing pages is not
* killed at the hard deadline.
*/
tick: (note: string) => void;
/** #5445: the label exclusion resolved once for this sweep (see loops-exclusion.ts). */
exclusion: LoopsExclusionPolicy;
/** Threads whose newest message falls in the recent window — LLM
* extraction candidates, enqueued (capped) after the sweep. */
extractCandidates: Array<{ slug: string; threadId: string; newestMs: number }>;
/** #5868: the run's state (grace holds are recorded on it) and the threads detection ran on. */
loopState?: GoogleSourceState;
processedThreads: Set<string>;
graceBackfillSeeded?: boolean;
}
export type ActivePack = { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;
function assertContained(dir: string, path: string): void {
if (!isWriteTargetContained(path, dir)) {
throw new Error(`Path escapes managed dir: "${path}"`);
}
}
/**
* Write a page's temp file privately: a stale `.tmp` left by a crash is
* removed first (never followed), then the temp file is created exclusively at
* 0600 and fchmod-ed past the umask before any byte lands. Renaming it over
* the page makes new and rewritten pages 0600.
*/
function writePrivateTemp(tmpPath: string, markdown: string): void {
let stale: ReturnType<typeof lstatSync> | null = null;
try { stale = lstatSync(tmpPath); } catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
if (stale?.isDirectory()) {
throw new Error(`Stale temporary path ${tmpPath} is a directory; remove it and re-run the sync.`);
}
if (stale) unlinkSync(tmpPath);
const buf = Buffer.from(markdown, 'utf-8');
const fd = openSync(tmpPath, 'wx', 0o600);
try {
fchmodSync(fd, 0o600);
let off = 0;
while (off < buf.length) {
const n = writeSync(fd, buf, off, buf.length - off);
if (n <= 0) throw new Error(`Short write to ${tmpPath} at offset ${off}/${buf.length}`);
off += n;
}
} finally {
closeSync(fd);
}
}
/** GBRA-133 (#7292, #8645): the checkout file with one stored state's `## Facts` fence put back, for `replaceableFile`; null when it has a fence of its own. */
function withStoredFactsFence(fileBytes: string, state: StoredBody): string | null {
const body = state.compiled_truth ?? '';
const start = body.indexOf(FACTS_FENCE_BEGIN);
const stop = start === -1 ? -1 : body.indexOf(FACTS_FENCE_END, start + FACTS_FENCE_BEGIN.length);
const block = stop === -1 ? null : body.slice(start, stop + FACTS_FENCE_END.length);
return block === null || hasFence(fileBytes) ? null : replaceOrInsertFactsFence(fileBytes, block);
}
export async function importRendered(
deps: GoogleSyncDeps,
relPath: string,
markdown: string,
activePack: ActivePack,
summary: GoogleSyncSummary,
countedSlugs: Set<string>,
identity: Record<string, string | readonly string[]> = {},
): Promise<string | null> {
markdown = connectorRender(markdown, { path: relPath, account: deps.cfg.account, ...identity });
const slug = slugifyPath(relPath);
const [labels] = await deps.engine.executeRaw<{ type: string; frontmatter: unknown }>(
'SELECT type, frontmatter FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL', [deps.sourceId, slug]);
if (labels && isPrivatePage(labels)) markdown = withPrivateVisibility(markdown);
if (deps.managed) {
const result = await deps.managed.importMarkdown(relPath, markdown);
if (result.purged) return null;
if (result.status === 'imported') {
summary.pagesAffected.push(result.slug);
summary.chunksCreated += result.chunks;
if (!countedSlugs.has(result.slug)) {
if (result.created) summary.added++; else summary.modified++;
countedSlugs.add(result.slug);
}
}
return result.slug;
}
const filePath = join(deps.cfg.dir, relPath);
assertContained(deps.cfg.dir, filePath);
const existing = await deps.engine.getPage(slug, { sourceId: deps.sourceId });
if (existing) assertInlineBody(existing, 'Google connector materialization');
markdown = await carryStoredFences(deps.sourceId, slug, relPath, existing, markdown, 'Google connector fence carry');
// #8644: content the owner purged is skipped before any file is touched, as the managed connector skips it (#6831).
if (await intentCarriesPurgedContent(deps.engine, { source_id: deps.sourceId, slug, intent: { content: markdown } }, { facts: false, pages: true })) {
deps.log(`[google] skipped ${slug}: its content matches a page purge tombstone; the rest of the sync continues. To let it back in: gbrain pages unpurge ${slug}`);
return null;
}
const databaseOnly = await isWriteThroughDisabled(deps.engine) || isSourceDbOnlySlug(deps.cfg.dir, slug, 'refuse');
const guard = databaseOnly ? null : await replaceableFile(deps.engine, deps.sourceId, slug, filePath, existing ? withStoredFactsFence : undefined);
const drift = () => new OperationError('file_database_drift',
`${slug}: the checkout file has an edit not retained in the database, or changed during import. The file was not replaced. If the file is newer, run gbrain sync --source ${deps.sourceId}; if the database is newer, re-save its page with write-through on; if both changed, ask the user which version to keep before retrying.`,
'Import the newer file with gbrain sync, or ask the user which version to keep; the item is held until then.');
if (guard && !guard.ok) throw drift();
const before = databaseOnly ? !!existing : existsSync(filePath);
// Temp-write → import → rename: a failed import never destroys the
// previously-good page (github-source pattern).
const tmpPath = `${filePath}.tmp`;
if (!databaseOnly) {
mkdirPrivate(dirname(filePath), deps.cfg.dir);
writePrivateTemp(tmpPath, markdown);
}
try {
const { importFile, importFromContent } = await import('../import-file.ts');
const opts = {
noEmbed: true, // embeds handled by the size gate below, like sync
sourceId: deps.sourceId,
...(activePack ? { activePack } : {}),
};
const result = databaseOnly
? await importFromContent(deps.engine, slug, markdown, { ...opts, sourcePath: relPath, preserveGateMarkers: true, holdNoncanonicalPrivate: true })
: await importFile(deps.engine, tmpPath, relPath, opts);
if (result.status === 'error' || result.error) {
throw new Error(result.error ?? `Import failed for ${relPath}`);
}
if (guard?.ok) {
const current = existsSync(filePath) ? readFileSync(filePath, 'utf8') : null;
if (current !== guard.expectedFileBytes) throw drift();
renameSync(tmpPath, filePath);
}
if (result.status === 'imported') {
summary.pagesAffected.push(result.slug);
summary.chunksCreated += result.chunks;
if (!countedSlugs.has(result.slug)) {
if (before) summary.modified++;
else summary.added++;
countedSlugs.add(result.slug);
}
}
return result.slug;
} finally {
if (!databaseOnly) rmSync(tmpPath, { force: true });
}
}
export async function deletePageByRelPath(
deps: GoogleSyncDeps,
relPath: string,
summary: GoogleSyncSummary,
): Promise<void> {
const rows = await deps.engine.executeRaw<{ slug: string }>(
`SELECT slug FROM pages WHERE source_id = $1 AND source_path = $2 AND deleted_at IS NULL`,
[deps.sourceId, relPath],
);
if (deps.managed) {
for (const row of rows) if (await deps.managed.delete(row.slug, relPath)) summary.deleted++;
return;
}
if (rows.length > 0) {
await deps.engine.deletePages(rows.map((r) => r.slug), { sourceId: deps.sourceId });
summary.deleted += rows.length;
}
// Same containment guard as the write path: a hostile/corrupt DB path
// carrying `../` must never unlink outside the managed dir.
const target = join(deps.cfg.dir, relPath);
if (isWriteTargetContained(target, deps.cfg.dir)) rmSync(target, { force: true });
}