Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.0 KB
Raw Blame
/**
 * sweep-shared — the per-run dependency bag and the page write/delete
 * primitives every Google sweep (contacts, calendar, Gmail) shares. Peeled
 * from google-source.ts so the sweeps can live in their own modules.
 */
import { closeSync, existsSync, fchmodSync, lstatSync, openSync, readFileSync, renameSync, rmSync, unlinkSync, writeSync } from 'node:fs';
import { dirname, join } from 'node:path';

import type { BrainEngine } from '../engine.ts';
import type { SyncOpts } from '../../commands/sync.ts';
import type { CredentialEntry } from '../creds/vault.ts';
import type { ManagedConnectorSync } from '../persistence/connector-sync.ts';
import { connectorRender } from '../connectors/connector-text.ts';
import { isWriteTargetContained } from '../path-confine.ts';
import { mkdirPrivate } from '../atomic-write.ts';
import { replaceableFile, type StoredBody } from '../facts/file-body-guard.ts';
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END, replaceOrInsertFactsFence } from '../facts-fence.ts';
import { carryStoredFences, hasFence } from '../connectors/fence-carry.ts';
import { assertInlineBody } from '../page-body/page-shape.ts';
import { OperationError } from '../ops/contract.ts';
import { slugifyPath } from '../sync.ts';
import { isPrivatePage } from '../search/private-visibility.ts';
import { withPrivateVisibility } from './google-render.ts';
import { isWriteThroughDisabled } from '../write-through.ts';
import { isSourceDbOnlySlug } from '../persistence/source-storage.ts';
import { intentCarriesPurgedContent } from '../persistence/purged-intent.ts';
import type { GoogleSourceConfig, GoogleSourceState } from './types.ts';
import type { LoopsExclusionPolicy } from './loops-exclusion.ts';

/** The "my addresses" identity set: account + Gmail sendAs aliases. */
export function myAddressSet(entry: CredentialEntry): Set<string> {
  const out = new Set<string>();
  if (entry.meta.account) out.add(entry.meta.account.toLowerCase());
  for (const a of entry.meta.sendas_aliases ?? []) out.add(a.toLowerCase());
  return out;
}

export interface GoogleSyncSummary {
  /** 'up_to_date'/'first_sync' are computed on the SyncResult, never here. */
  status: 'synced' | 'partial';
  added: number;
  modified: number;
  deleted: number;
  chunksCreated: number;
  embedded: number;
  pagesAffected: string[];
  threadsSeen: number;
  attachmentInspection: Record<string, number>;
  /**
   * Why each in-window thread was or was not sent to the extractor, keyed by
   * the machine reason from loopExtractionEligibility. Counts only — no
   * addresses, subjects or body text — so a sweep can be audited for
   * over-filtering without leaking mail content into logs.
   */
  extractEligibility: Record<string, number>;
  /** #5012: reply-owed loops the detector withheld this sweep, by reason (counts only). */
  loopsWithheld: Record<string, number>;
  failedFiles: number;
}

export interface GoogleSyncDeps {
  managed: ManagedConnectorSync | null;
  engine: BrainEngine;
  sourceId: string;
  cfg: GoogleSourceConfig;
  opts: SyncOpts;
  entry: CredentialEntry;
  log: (msg: string) => void;
  /**
   * #5349: one forward-progress note per imported item. The progress-aware
   * sync deadline extends only on these (heartbeats are not progress), so a
   * long first contacts or calendar sweep that is still landing pages is not
   * killed at the hard deadline.
   */
  tick: (note: string) => void;
  /** #5445: the label exclusion resolved once for this sweep (see loops-exclusion.ts). */
  exclusion: LoopsExclusionPolicy;
  /** Threads whose newest message falls in the recent window — LLM
   *  extraction candidates, enqueued (capped) after the sweep. */
  extractCandidates: Array<{ slug: string; threadId: string; newestMs: number }>;
  /** #5868: the run's state (grace holds are recorded on it) and the threads detection ran on. */
  loopState?: GoogleSourceState;
  processedThreads: Set<string>;
  graceBackfillSeeded?: boolean;
}

export type ActivePack = { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }> } | undefined;

function assertContained(dir: string, path: string): void {
  if (!isWriteTargetContained(path, dir)) {
    throw new Error(`Path escapes managed dir: "${path}"`);
  }
}

/**
 * Write a page's temp file privately: a stale `.tmp` left by a crash is
 * removed first (never followed), then the temp file is created exclusively at
 * 0600 and fchmod-ed past the umask before any byte lands. Renaming it over
 * the page makes new and rewritten pages 0600.
 */
function writePrivateTemp(tmpPath: string, markdown: string): void {
  let stale: ReturnType<typeof lstatSync> | null = null;
  try { stale = lstatSync(tmpPath); } catch (error) {
    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
  }
  if (stale?.isDirectory()) {
    throw new Error(`Stale temporary path ${tmpPath} is a directory; remove it and re-run the sync.`);
  }
  if (stale) unlinkSync(tmpPath);
  const buf = Buffer.from(markdown, 'utf-8');
  const fd = openSync(tmpPath, 'wx', 0o600);
  try {
    fchmodSync(fd, 0o600);
    let off = 0;
    while (off < buf.length) {
      const n = writeSync(fd, buf, off, buf.length - off);
      if (n <= 0) throw new Error(`Short write to ${tmpPath} at offset ${off}/${buf.length}`);
      off += n;
    }
  } finally {
    closeSync(fd);
  }
}


/** GBRA-133 (#7292, #8645): the checkout file with one stored state's `## Facts` fence put back, for `replaceableFile`; null when it has a fence of its own. */
function withStoredFactsFence(fileBytes: string, state: StoredBody): string | null {
  const body = state.compiled_truth ?? '';
  const start = body.indexOf(FACTS_FENCE_BEGIN);
  const stop = start === -1 ? -1 : body.indexOf(FACTS_FENCE_END, start + FACTS_FENCE_BEGIN.length);
  const block = stop === -1 ? null : body.slice(start, stop + FACTS_FENCE_END.length);
  return block === null || hasFence(fileBytes) ? null : replaceOrInsertFactsFence(fileBytes, block);
}

export async function importRendered(
  deps: GoogleSyncDeps,
  relPath: string,
  markdown: string,
  activePack: ActivePack,
  summary: GoogleSyncSummary,
  countedSlugs: Set<string>,
  identity: Record<string, string | readonly string[]> = {},
): Promise<string | null> {
  markdown = connectorRender(markdown, { path: relPath, account: deps.cfg.account, ...identity });
  const slug = slugifyPath(relPath);
  const [labels] = await deps.engine.executeRaw<{ type: string; frontmatter: unknown }>(
    'SELECT type, frontmatter FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL', [deps.sourceId, slug]);
  if (labels && isPrivatePage(labels)) markdown = withPrivateVisibility(markdown);
  if (deps.managed) {
    const result = await deps.managed.importMarkdown(relPath, markdown);
    if (result.purged) return null;
    if (result.status === 'imported') {
      summary.pagesAffected.push(result.slug);
      summary.chunksCreated += result.chunks;
      if (!countedSlugs.has(result.slug)) {
        if (result.created) summary.added++; else summary.modified++;
        countedSlugs.add(result.slug);
      }
    }
    return result.slug;
  }
  const filePath = join(deps.cfg.dir, relPath);
  assertContained(deps.cfg.dir, filePath);
  const existing = await deps.engine.getPage(slug, { sourceId: deps.sourceId });
  if (existing) assertInlineBody(existing, 'Google connector materialization');
  markdown = await carryStoredFences(deps.sourceId, slug, relPath, existing, markdown, 'Google connector fence carry');
  // #8644: content the owner purged is skipped before any file is touched, as the managed connector skips it (#6831).
  if (await intentCarriesPurgedContent(deps.engine, { source_id: deps.sourceId, slug, intent: { content: markdown } }, { facts: false, pages: true })) {
    deps.log(`[google] skipped ${slug}: its content matches a page purge tombstone; the rest of the sync continues. To let it back in: gbrain pages unpurge ${slug}`);
    return null;
  }
  const databaseOnly = await isWriteThroughDisabled(deps.engine) || isSourceDbOnlySlug(deps.cfg.dir, slug, 'refuse');
  const guard = databaseOnly ? null : await replaceableFile(deps.engine, deps.sourceId, slug, filePath, existing ? withStoredFactsFence : undefined);
  const drift = () => new OperationError('file_database_drift',
    `${slug}: the checkout file has an edit not retained in the database, or changed during import. The file was not replaced. If the file is newer, run gbrain sync --source ${deps.sourceId}; if the database is newer, re-save its page with write-through on; if both changed, ask the user which version to keep before retrying.`,
    'Import the newer file with gbrain sync, or ask the user which version to keep; the item is held until then.');
  if (guard && !guard.ok) throw drift();
  const before = databaseOnly ? !!existing : existsSync(filePath);
  // Temp-write → import → rename: a failed import never destroys the
  // previously-good page (github-source pattern).
  const tmpPath = `${filePath}.tmp`;
  if (!databaseOnly) {
    mkdirPrivate(dirname(filePath), deps.cfg.dir);
    writePrivateTemp(tmpPath, markdown);
  }
  try {
    const { importFile, importFromContent } = await import('../import-file.ts');
    const opts = {
      noEmbed: true, // embeds handled by the size gate below, like sync
      sourceId: deps.sourceId,
      ...(activePack ? { activePack } : {}),
    };
    const result = databaseOnly
      ? await importFromContent(deps.engine, slug, markdown, { ...opts, sourcePath: relPath, preserveGateMarkers: true, holdNoncanonicalPrivate: true })
      : await importFile(deps.engine, tmpPath, relPath, opts);
    if (result.status === 'error' || result.error) {
      throw new Error(result.error ?? `Import failed for ${relPath}`);
    }
    if (guard?.ok) {
      const current = existsSync(filePath) ? readFileSync(filePath, 'utf8') : null;
      if (current !== guard.expectedFileBytes) throw drift();
      renameSync(tmpPath, filePath);
    }
    if (result.status === 'imported') {
      summary.pagesAffected.push(result.slug);
      summary.chunksCreated += result.chunks;
      if (!countedSlugs.has(result.slug)) {
        if (before) summary.modified++;
        else summary.added++;
        countedSlugs.add(result.slug);
      }
    }
    return result.slug;
  } finally {
    if (!databaseOnly) rmSync(tmpPath, { force: true });
  }
}

export async function deletePageByRelPath(
  deps: GoogleSyncDeps,
  relPath: string,
  summary: GoogleSyncSummary,
): Promise<void> {
  const rows = await deps.engine.executeRaw<{ slug: string }>(
    `SELECT slug FROM pages WHERE source_id = $1 AND source_path = $2 AND deleted_at IS NULL`,
    [deps.sourceId, relPath],
  );
  if (deps.managed) {
    for (const row of rows) if (await deps.managed.delete(row.slug, relPath)) summary.deleted++;
    return;
  }
  if (rows.length > 0) {
    await deps.engine.deletePages(rows.map((r) => r.slug), { sourceId: deps.sourceId });
    summary.deleted += rows.length;
  }
  // Same containment guard as the write path: a hostile/corrupt DB path
  // carrying `../` must never unlink outside the managed dir.
  const target = join(deps.cfg.dir, relPath);
  if (isWriteTargetContained(target, deps.cfg.dir)) rmSync(target, { force: true });
}