Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.7 KB
Raw Blame
/**
 * Shared machinery of the API-backed connector sweeps (the meetings and slack
 * kinds, autoplan T30): the coverage window and its committed boundary, the
 * provider rate-limit record the scheduler reads, and the publication of one
 * page in either record mode.
 *
 * Coverage: a run works one window [from, to). `to` is pinned to the run
 * start and persisted with the window, so an interrupted run resumes the same
 * window before a new one opens. The boundary (`committed_to`) advances to
 * `to` only when the caller has covered the whole window. The next window
 * starts `lookbackDays` before the boundary, so late edits and deletions in
 * recent items converge.
 *
 * Records (D5): `git` mode commits each page through the door before its
 * database row (managed: the coordinator; unmanaged: the commit-push door,
 * then the import); `database` mode imports database-only with no directory.
 * A private page stays database-only on a Git-recorded source, and one that
 * became private leaves the checkout. A page whose content changed upstream,
 * or that was deleted upstream, retires the facts extracted from it.
 */
import { join, relative } from 'node:path';
import { existsSync, mkdirSync, realpathSync, rmSync } from 'node:fs';
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts, SyncResult } from '../../commands/sync.ts';
import { OperationError } from '../ops/contract.ts';
import type { ManagedConnectorSync } from '../persistence/connector-sync.ts';
import { carryStoredFences } from './fence-carry.ts';
import { ConnectorAccessDeniedError, ConnectorAuthError, ConnectorRateLimitedError, ConnectorRunLimitError } from './client.ts';
import { atomicWriteFileSync } from '../atomic-write.ts';
import { isWriteTargetContained } from '../path-confine.ts';
import { slugifyPath } from '../sync.ts';
import { isSegmentedPage } from '../page-body/page-shape.ts';
import { connectorFactsSession, retireFactsFromPage } from '../facts/retire.ts';

export const DAY_MS = 86_400_000;

export interface CoverageWindow { from: string; to: string; done: string[] }
export interface CoverageState { committed_to: string | null; window: CoverageWindow | null }

/**
 * The window this run works: the persisted one when an earlier run did not finish it, else
 * [boundary - lookback, runStart) after the first sync and [runStart - history, runStart) before it.
 * `alignFrom` moves `from` back to a boundary the caller needs (the start of a day page).
 */
export function openCoverageWindow(state: CoverageState, opts: { runStart: number; historyDays: number; lookbackDays: number; alignFrom?: (ms: number) => number }): CoverageWindow {
  if (state.window) return state.window;
  const from = state.committed_to ? Date.parse(state.committed_to) - opts.lookbackDays * DAY_MS : opts.runStart - opts.historyDays * DAY_MS;
  state.window = { from: new Date(opts.alignFrom ? opts.alignFrom(from) : from).toISOString(), to: new Date(opts.runStart).toISOString(), done: [] };
  return state.window;
}

/** The window is covered: the boundary moves to its end. Returns the horizon before which no later window reaches. */
export function closeCoverageWindow(state: CoverageState, lookbackDays: number): number {
  const to = state.window!.to;
  state.committed_to = to;
  state.window = null;
  return Date.parse(to) - (lookbackDays + 1) * DAY_MS;
}

/** An unmanaged dry run: these sources fetch and publish in one pass, so a preview calls no provider and writes nothing. */
export function connectorDryRunResult(label: string, sourceId: string): SyncResult {
  process.stderr.write(`[sync] ${sourceId}: a ${label} source has no preview; nothing was fetched or written. Run gbrain sync --source ${sourceId} without --dry-run to sync it.\n`);
  return { status: 'dry_run', fromCommit: null, toCommit: '', added: 0, modified: 0, deleted: 0, renamed: 0, chunksCreated: 0, embedded: 0, pagesAffected: [] };
}

/** Errors that stop the whole run (credential, permission, quota, budget, cancel) rather than hold one item. */
export function runScoped(error: unknown): boolean {
  const name = (error as { name?: string } | null)?.name;
  return error instanceof ConnectorAuthError || error instanceof ConnectorAccessDeniedError || error instanceof ConnectorRateLimitedError
    || error instanceof ConnectorRunLimitError || name === 'AbortError' || name === 'TimeoutError' || name === 'LockStolenError' || name === 'ConnectorWaitBudgetStop';
}

/** Records the provider's reset time so the scheduler does not resubmit before it (D1). */
export async function recordRateLimit(engine: BrainEngine, sourceId: string, retryAt: number): Promise<void> {
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('connector-rate-limit',$1,$2::text::jsonb)
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys,updated_at=now()`,
  [sourceId, JSON.stringify([{ version: 1, retry_at: new Date(retryAt).toISOString() }])]).catch(() => undefined);
}

export function connectorBindingMissing(label: string, sourceId: string): OperationError {
  return new OperationError('connector_binding_missing',
    `${label} source ${sourceId} records to Git, but its checkout is not available; the item was held, not written database-only.`,
    `Restore the source's Git checkout (gbrain sources status ${sourceId}), then retry the held items: gbrain sources retry-held ${sourceId}. To record this source in the database instead, change its record mode explicitly.`,
    'docs/guides/meeting-and-slack-sources.md#connector_binding_missing');
}

/** A page path inside the source directory, as the checkout root names it (the commit door takes checkout-relative paths). */
function checkoutPath(root: string, dir: string, relPath: string): string {
  return relative(realpathSync(root), join(realpathSync(dir), relPath)).split('\\').join('/');
}

/** The git checkout that holds an unmanaged git-mode source's directory, or null. */
export async function gitCheckoutOf(dir: string | null): Promise<string | null> {
  if (!dir || !existsSync(dir)) return null;
  try {
    const { checkoutRoot } = await import('../git/commit-push.ts');
    return checkoutRoot(dir);
  } catch { return null; }
}

export interface PublishSummary {
  added: number; modified: number; deleted: number; chunksCreated: number; embedded: number; pagesAffected: string[]; factsRetired: number;
}

export interface ConnectorPublishCtx {
  engine: BrainEngine; sourceId: string; managed: ManagedConnectorSync | null; summary: PublishSummary;
  /** The kind as people name it in messages ("Meetings", "Slack"), and the lower-case tag in commit subjects and logs. */
  label: string; tag: string;
  record: 'git' | 'database'; dir: string | null;
  /** An unmanaged import passes the external-untrusted write gate, as every managed connector import does. */
  gated?: boolean;
}

export interface ConnectorPage {
  relPath: string;
  /** Rendered through `connectorRender`. */
  markdown: string;
  /** Commit message when the page is committed; the subject when a page that became private leaves the checkout. */
  commitMessage: string;
  privateMessage: string;
  /** Recorded on the retired facts when the page changed. */
  retireReason: string;
}

/** Publishes one page in the source's record mode; counts it and retires the facts of a changed page. */
export async function publishConnectorPage(ctx: ConnectorPublishCtx, page: ConnectorPage): Promise<'added' | 'modified' | 'unchanged'> {
  const { engine, sourceId, managed, summary } = ctx;
  const { relPath } = page;
  const slug = slugifyPath(relPath);
  let markdown = page.markdown;
  let status: 'imported' | 'skipped';
  let created: boolean;
  if (managed) {
    if (ctx.record === 'git' && !managed.bound) throw connectorBindingMissing(ctx.label, sourceId);
    const result = await managed.importMarkdown(relPath, markdown);
    status = result.status; created = result.created;
    summary.chunksCreated += result.chunks;
  } else {
    const existing = await engine.getPage(slug, { sourceId });
    markdown = await carryStoredFences(sourceId, slug, relPath, existing, markdown, `${ctx.tag} connector fence carry`);
    const isPrivate = /^visibility: private$/m.test(markdown.split('\n---\n')[0] ?? '');
    if (ctx.record === 'git') {
      const root = await gitCheckoutOf(ctx.dir);
      if (!root || !ctx.dir) throw connectorBindingMissing(ctx.label, sourceId);
      const filePath = join(ctx.dir, relPath);
      if (!isWriteTargetContained(filePath, ctx.dir)) throw new OperationError('invalid_params', `${relPath} resolves outside the source directory.`, 'This is a gbrain defect; report it.');
      const { commitPathsLocked } = await import('../git/commit-push.ts');
      if (isPrivate) {
        if (existsSync(filePath)) { rmSync(filePath); commitPathsLocked(ctx.dir, [checkoutPath(root, ctx.dir, relPath)], page.privateMessage, { lockWaitMs: 30_000 }); }
      } else {
        mkdirSync(join(filePath, '..'), { recursive: true, mode: 0o700 });
        atomicWriteFileSync(filePath, markdown, { mode: 0o600 });
        commitPathsLocked(ctx.dir, [checkoutPath(root, ctx.dir, relPath)], page.commitMessage, { lockWaitMs: 30_000 });
      }
    }
    const { importFromContent } = await import('../import-file.ts');
    const result = await importFromContent(engine, slug, markdown, { noEmbed: true, sourceId, sourcePath: relPath, preserveGateMarkers: true, holdNoncanonicalPrivate: true,
      ...(ctx.gated ? { writeGate: { tier: 'external_untrusted' as const, origin: { channel: `connector:${ctx.tag}`, connector: ctx.tag, source_uri: relPath }, requestId: null } } : {}) });
    if (result.status === 'error' || result.error) throw new Error(result.error ?? `Import failed for ${relPath}`);
    status = result.status === 'imported' ? 'imported' : 'skipped';
    created = !existing;
    summary.chunksCreated += result.chunks;
  }
  if (status !== 'imported') return 'unchanged';
  summary.pagesAffected.push(slug);
  if (created) { summary.added++; return 'added'; }
  summary.modified++;
  const retired = await retireFactsFromPage(engine, { sourceId, slug, reason: page.retireReason, managed: !!managed });
  summary.factsRetired += retired.retired;
  return 'modified';
}

/** Deletes one page whose item was deleted upstream, after retiring its facts. */
export async function deleteConnectorPage(ctx: ConnectorPublishCtx, relPath: string, commitMessage: string, retireReason: string): Promise<void> {
  const { engine, sourceId, managed, summary } = ctx;
  const slug = slugifyPath(relPath);
  const retired = await retireFactsFromPage(engine, { sourceId, slug, reason: retireReason, managed: !!managed });
  summary.factsRetired += retired.retired;
  if (managed) {
    if (await managed.delete(slug, relPath)) summary.deleted++;
    return;
  }
  if (ctx.record === 'git' && ctx.dir) {
    const filePath = join(ctx.dir, relPath);
    const root = await gitCheckoutOf(ctx.dir);
    if (!root) throw connectorBindingMissing(ctx.label, sourceId);
    if (isWriteTargetContained(filePath, ctx.dir) && existsSync(filePath)) {
      rmSync(filePath);
      const { commitPathsLocked } = await import('../git/commit-push.ts');
      commitPathsLocked(ctx.dir, [checkoutPath(root, ctx.dir, relPath)], commitMessage, { lockWaitMs: 30_000 });
    }
  }
  const rows = await engine.executeRaw<{ slug: string }>('SELECT slug FROM pages WHERE source_id=$1 AND slug=$2 AND deleted_at IS NULL', [sourceId, slug]);
  if (rows.length) { await engine.deletePages([slug], { sourceId }); summary.deleted++; }
}

/** A connector run queues fact extraction for at most this many pages; the cycle's backfill covers the rest. */
export const CONNECTOR_FACTS_PAGE_LIMIT = 50;

/** Facts and embeddings for the pages a run published (the connector path has no sync finalize). */
export async function afterConnectorPublish(ctx: Pick<ConnectorPublishCtx, 'engine' | 'sourceId' | 'summary' | 'managed' | 'tag'>, opts: SyncOpts): Promise<void> {
  const { engine, sourceId, summary, managed } = ctx;
  if (!summary.pagesAffected.length) return;
  const overLimit = summary.pagesAffected.length > CONNECTOR_FACTS_PAGE_LIMIT;
  // #9552: on a managed brain each import's publication queued its own fact extraction (connector-sync.ts).
  if (!opts.noExtract && !overLimit && !managed) {
    const { runFactsBackstop } = await import('../facts/backstop.ts');
    let failed = 0;
    let firstError = '';
    for (const slug of summary.pagesAffected) {
      try {
        const page = await engine.getPage(slug, { sourceId });
        if (!page || isSegmentedPage(page)) continue;
        await runFactsBackstop({ slug, type: page.type, compiled_truth: page.compiled_truth ?? '', frontmatter: page.frontmatter ?? {} },
          { engine, sourceId, sessionId: connectorFactsSession(sourceId, slug), source: 'sync:import', mode: 'queue', notabilityFilter: 'high-only', sourceSlug: slug });
      } catch (error) {
        failed++;
        firstError ||= error instanceof Error ? error.message : String(error);
        const { writeFactsAbsorbFailure } = await import('../facts/absorb-log.ts');
        await writeFactsAbsorbFailure(engine, slug, error, sourceId);
      }
    }
    if (failed) process.stderr.write(`[${ctx.tag}] ${sourceId}: fact extraction was not queued for ${failed} page(s): ${firstError}\n`);
  } else if (!opts.noExtract && overLimit) {
    process.stderr.write(`[${ctx.tag}] ${summary.pagesAffected.length} pages changed; ${managed ? `facts were queued for the first ${CONNECTOR_FACTS_PAGE_LIMIT}, the rest` : 'facts'} are extracted by the cycle's backfill\n`);
  }
  if (managed || opts.noEmbed || summary.pagesAffected.length > 100) return;
  try {
    const { runEmbedCore } = await import('../../commands/embed.ts');
    await runEmbedCore(engine, { slugs: summary.pagesAffected, sourceId });
    summary.embedded = summary.pagesAffected.length;
  } catch { /* embed is best-effort */ }
}