Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.6 KB
Raw Blame
/**
 * commands/connectors/sync.ts — `gbrain connectors sync <provider>|--all [flags]`.
 *
 * Inline by default (PGLite-safe: it's the same runConnectorSync the handler
 * wraps). `--background` submits the `connector-sync` minion job on Postgres;
 * on PGLite (no worker daemon) it falls back to inline with a note.
 *
 * #7515: `--all` (the scheduled form) syncs only providers whose
 * `connectors.<p>.auto_sync` is on; an explicit provider overrides it.
 * `--max-requests` and `--deadline` bound the whole invocation under the hard
 * MAX_CONNECTOR_REQUESTS_PER_RUN ceiling every run has.
 */

import type { BrainEngine } from '../../core/engine.ts';
import type { CliDispatchContext } from '../../cli/command-table.ts';
import { setCliExitVerdict } from '../../core/cli-force-exit.ts';
import { createProgress } from '../../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../../core/cli-options.ts';
import { runConnectorSync } from '../../core/connectors/sync.ts';
import type { ConnectorSyncResult } from '../../core/connectors/sync.ts';
import { connectorProviderNames, isConnectorProviderName } from '../../core/connectors/registry.ts';
import { loadCredential } from '../../core/connectors/credentials.ts';
import { autoSyncKey, isTruthy, sourceIdKey } from '../../core/connectors/config-keys.ts';
import { MAX_CONNECTOR_REQUESTS_PER_RUN } from '../../core/connectors/client.ts';
import { parseDurationSeconds } from '../../core/sync-concurrency.ts';
import type { ConnectorProviderName } from '../../core/connectors/types.ts';
import { flagValueError, intFlagValue } from '../../cli/flag-values.ts';
import { usageError } from '../../cli/cli-error.ts';

interface SyncFlags {
  full: boolean;
  dryRun: boolean;
  limit?: number;
  windowDays?: number;
  source?: string;
  embed: boolean;
  background: boolean;
  json: boolean;
  all: boolean;
  maxRequests?: number;
  deadlineSec?: number;
}

function deadlineFlagValue(raw: string | undefined): number {
  const accepts = 'a duration such as 90s, 30m or 2h';
  try {
    const seconds = raw === undefined || raw.startsWith('-') ? undefined : parseDurationSeconds(raw, '--deadline');
    if (seconds !== undefined) return seconds;
  } catch {
    // falls through to the usage error below
  }
  throw flagValueError('--deadline', raw, accepts, '30m');
}

function parseFlags(args: string[]): { provider: string; providerIndex: number; flags: SyncFlags } {
  const flags: SyncFlags = { full: false, dryRun: false, embed: false, background: false, json: false, all: false };
  let provider = '';
  let providerIndex = -1;
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--all') flags.all = true;
    else if (a === '--full') flags.full = true;
    else if (a === '--dry-run') flags.dryRun = true;
    else if (a === '--embed') flags.embed = true;
    else if (a === '--background') flags.background = true;
    else if (a === '--json') flags.json = true;
    // #5930 (D4): decimals, zero limits and unsafe integers are usage errors (exit 2), not Number() coercions.
    else if (a === '--limit') flags.limit = intFlagValue(args[++i], '--limit', { min: 1, example: 50 });
    else if (a === '--window-days') flags.windowDays = intFlagValue(args[++i], '--window-days', { min: 0, example: 30 });
    else if (a === '--max-requests') flags.maxRequests = intFlagValue(args[++i], '--max-requests', { min: 1, max: MAX_CONNECTOR_REQUESTS_PER_RUN, example: 500 });
    else if (a === '--deadline') flags.deadlineSec = deadlineFlagValue(args[++i]);
    else if (a === '--source') flags.source = args[++i];
    else if (!a.startsWith('-')) { provider = a; providerIndex = i; }
  }
  return { provider, providerIndex, flags };
}

const SYNC_USAGE = 'gbrain connectors sync <chatgpt|claude>|--all [--full] [--dry-run] [--limit N] [--window-days N] [--max-requests N] [--deadline 30m] [--source id] [--embed] [--background] [--json]';

/** Agent contract v1: usage errors are `invalid_params` (exit 2, one `--json` envelope) naming the usage and the fix. */
function syncUsageError(message: string, fix: { argv: string[]; inputs: Array<{ name: string; how: string }> }) {
  return usageError(message, `Usage: ${SYNC_USAGE}`, {
    why: 'gbrain connectors sync needs one provider (or --all) and a value after every value flag.',
    fix: { ...fix, consent: [], actor: 'agent', why: 'A corrected command line runs the sync.', requires_exclusive: false },
  });
}

export async function runConnectorSyncCmd(engine: BrainEngine, args: string[], dispatch: Pick<CliDispatchContext, 'makeContext'> = {}): Promise<void> {
  if (args.includes('--help') || args.includes('-h')) {
    console.log(`Usage: ${SYNC_USAGE}`);
    return;
  }
  const badSource = args.findIndex((arg, i) => arg === '--source' && (!args[i + 1] || args[i + 1].startsWith('-')));
  if (badSource !== -1) {
    const argv = ['gbrain', 'connectors', 'sync', ...args.slice(0, badSource + 1), '<SOURCE_ID>', ...args.slice(badSource + 1)];
    throw syncUsageError(flagValueError('--source', args[badSource + 1], 'a source id', 'default').message, {
      argv, inputs: [{ name: 'SOURCE_ID', how: 'The id of the source the imported conversations belong to (gbrain sources list), or drop --source to use the configured connector source.' }],
    });
  }

  const { provider, providerIndex, flags } = parseFlags(args);

  // Resolve the target provider set.
  let providers: ConnectorProviderName[];
  if (flags.all) {
    // Only providers that have a credential (env or file) are worth syncing.
    const credentialed = connectorProviderNames().filter((p) => loadCredential(p) !== null);
    if (credentialed.length === 0) {
      console.error('No connector credentials found. Run `gbrain connectors auth <provider>` first.');
      setCliExitVerdict(1);
      return;
    }
    // #7515: --all is the scheduled form, so a provider whose owner turned auto_sync off is left alone.
    providers = [];
    for (const p of credentialed) {
      if (isTruthy(await engine.getConfig(autoSyncKey(p)))) providers.push(p);
      else console.error(`Skipped ${p}: connectors.${p}.auto_sync is off (\`gbrain connectors sync ${p}\` syncs it explicitly).`);
    }
  } else if (isConnectorProviderName(provider)) {
    providers = [provider];
  } else {
    const names = connectorProviderNames();
    throw syncUsageError(provider ? `Unknown connector provider '${provider}'; expected ${names.join(', ')} or --all.` : `Name a connector provider (${names.join(', ')}) or pass --all.`, {
      argv: ['gbrain', 'connectors', 'sync', '<PROVIDER>', ...args.filter((_, i) => i !== providerIndex)],
      inputs: [{ name: 'PROVIDER', how: `One of ${names.join(', ')}, or --all for every provider with a credential (gbrain connectors providers lists them).` }],
    });
  }

  const sourceId = flags.source ?? (await engine.getConfig(sourceIdKey())) ?? 'default';
  const results: ConnectorSyncResult[] = [];
  const deadlineAt = flags.deadlineSec !== undefined ? Date.now() + flags.deadlineSec * 1000 : undefined;

  for (const p of providers) {
    if (flags.background && !flags.dryRun) {
      const submitted = await submitBackground(engine, p, sourceId, flags);
      if (submitted) continue; // job queued (Postgres); else fell through to inline
    }
    const reporter = createProgress(cliOptsToProgressOptions(getCliOptions()));
    reporter.start(`connector.sync.${p}`);
    try {
      const r = await runConnectorSync(engine, {
        context: flags.dryRun ? undefined : await dispatch.makeContext?.(engine, { source: sourceId }),
        provider: p,
        sourceId,
        full: flags.full,
        dryRun: flags.dryRun,
        limit: flags.limit,
        windowDays: flags.windowDays,
        embed: flags.embed,
        maxRequests: flags.maxRequests,
        deadlineAt,
        onProgress: (pr) => reporter.heartbeat(`${pr.phase}: listed ${pr.listed}, fetched ${pr.fetched}, imported ${pr.imported}`),
      });
      results.push(r);
      reporter.finish();
      if (!flags.json) printResult(r);
    } catch (e) {
      reporter.finish();
      console.error(`connector sync ${p} failed: ${e instanceof Error ? e.message : String(e)}`);
      setCliExitVerdict(1);
    }
  }

  if (flags.json) console.log(JSON.stringify({ results }, null, 2));
}

async function submitBackground(
  engine: BrainEngine,
  provider: ConnectorProviderName,
  sourceId: string,
  flags: SyncFlags,
): Promise<boolean> {
  if (engine.kind !== 'postgres') {
    console.error(`(${provider}) --background needs Postgres (PGLite has no worker daemon); running inline.`);
    return false;
  }
  const { MinionQueue } = await import('../../core/minions/queue.ts');
  const queue = new MinionQueue(engine);
  const slot = new Date().toISOString().slice(0, 13); // hour bucket
  const job = await queue.add(
    'connector-sync',
    {
      provider, sourceId, full: flags.full, limit: flags.limit, maxRequests: flags.maxRequests,
      deadlineSec: flags.deadlineSec, explicit: !flags.all,
    },
    { idempotency_key: `connector-sync:${provider}:${slot}`, max_attempts: 2 },
  );
  console.log(`Queued connector-sync for ${provider} (job ${job.id}). Track with \`gbrain jobs get ${job.id}\`.`);
  return true;
}

function printResult(r: ConnectorSyncResult): void {
  const i = r.ingest;
  console.log(
    `${r.provider}: ${r.status}` +
      `  listed=${r.listed} fetched=${r.fetched} errors=${r.fetchErrors}` +
      (i ? `  imported=${i.imported} skipped=${i.skipped} redactions=${i.redactions}` : '') +
      (r.watermarkAdvancedTo ? `  watermark→${r.watermarkAdvancedTo}` : '') +
      (r.embedKickoff !== 'none' && r.embedKickoff !== 'below_threshold' ? `  embed:${r.embedKickoff}` : ''),
  );
  if (r.hint) console.log(`  ${r.hint.split('\n')[0]}`);
}