Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.8 KB
Raw Blame
/**
 * The slack source sweep (`sources.config.kind = 'slack'`): one page per
 * channel per day.
 *
 * Coverage is per channel (`connectors/api-source.ts` windows), so a channel
 * the app can no longer read is held without stopping the others. Each run
 * reads a channel's window [from, to): `from` is aligned to the start of a day
 * in the source's time zone, so every day the window touches is read whole and
 * its page is rebuilt from scratch, never merged with a delta. A day whose
 * top-level messages are unchanged since its last publish (the fingerprint)
 * costs one history read and no reply reads; `--full` rebuilds every day.
 * After the window, the next one starts `lookbackDays` before the boundary, so
 * edits, deletions and new replies to threads started in the lookback converge.
 * A reply to a thread that started before the lookback is not seen (a disclosed
 * coverage limit); a reply also sent to the channel is.
 *
 * Privacy: a private channel's pages are `visibility: private`, database-only
 * on a Git-recorded source. A channel that becomes private has every page of
 * it re-published private, its committed files removed at HEAD, and the facts
 * extracted from it retired (T23).
 */
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { SyncOpts, SyncResult } from '../../commands/sync.ts';
import { OperationError } from '../ops/contract.ts';
import { withConnectorSync, pendingConnectorResult, type ManagedConnectorSync } from '../persistence/connector-sync.ts';
import { connectorIdentity } from '../persistence/connector-identity.ts';
import { connectorAccountChanged } from '../persistence/connector-errors.ts';
import { sameConnectorAccount, type ConnectorAccount } from '../persistence/connector-state.ts';
import { ConnectorHoldSession, connectorHoldsResult } from '../connectors/connector-hold-session.ts';
import { connectorAccountPinKey, connectorDbCursorKey, readConnectorDbCursor, writeConnectorDbCursor } from '../connectors/db-cursor.ts';
import { resolveApiToken } from '../connectors/credentials.ts';
import { connectorRender } from '../connectors/connector-text.ts';
import { serializePageToMarkdown } from '../markdown.ts';
import { assertInlineBody } from '../page-body/page-shape.ts';
import { ConnectorAccessDeniedError, ConnectorAuthError, ConnectorRateLimitedError, ConnectorRunLimitError } from '../connectors/client.ts';
import {
  afterConnectorPublish, closeCoverageWindow, connectorDryRunResult, deleteConnectorPage, openCoverageWindow, publishConnectorPage, recordRateLimit, runScoped,
  type ConnectorPublishCtx, type CoverageWindow, type PublishSummary,
} from '../connectors/api-source.ts';
import { slugifyPath } from '../sync.ts';
import { VERSION } from '../../version.ts';
import { SlackChannelUnavailableError, slackApi, type SlackApi } from './client.ts';
import { nextSlackDay, renderSlackDay, skippedMessage, slackAlignToDay, slackDayFingerprint, slackDayKey, slackDayStartMs, slackPagePath } from './render.ts';
import type { SlackAccount, SlackChannelInfo, SlackMessage, SlackSourceConfig, SlackUser } from './types.ts';

/** The user list is re-read when a message names a user it lacks, or once this old. */
const USERS_MAX_AGE_MS = 7 * 86_400_000;
// A user the last listing lacked (cached under its raw id) is looked up again after a day; a Slack Connect user never appears.
const USERS_MISSING_RETRY_MS = 86_400_000;

export interface SlackChannelState {
  committed_to: string | null;
  window: CoverageWindow | null;
  name: string;
  private: boolean;
  /** Days published in the lookback horizon: the page path and the fingerprint it was rendered from. */
  days: Record<string, { path: string; fp: string }>;
}

export interface SlackState extends Record<string, unknown> {
  channels: Record<string, SlackChannelState>;
  /** Display names of the users the pages mention (T27): refreshed by one users.list at most per run. */
  users: Record<string, SlackUser>;
  users_listed_at: string | null;
  account?: ConnectorAccount | null;
  item_holds?: unknown;
}

const emptyState = (): SlackState => ({ channels: {}, users: {}, users_listed_at: null });

export interface SlackSyncDeps {
  /** Test seam: an API bound to a fake transport. */
  api?: SlackApi;
  now?: () => number;
}

interface Summary extends PublishSummary { status: 'synced' | 'partial'; failedFiles: number }

export function slackConnectorAccount(account: SlackAccount): ConnectorAccount {
  return { kind: 'slack', teamId: account.teamId, enterpriseId: account.enterpriseId };
}

export function slackAuthRequired(sourceId: string, cfg: Pick<SlackSourceConfig, 'tokenEnv'>, detail?: string): OperationError {
  return new OperationError('connector_auth_required',
    `Slack source ${sourceId} has no working Slack app token${detail ? ` (${detail})` : ''}. Nothing was imported.`,
    cfg.tokenEnv ? `Set ${cfg.tokenEnv} to the app's Bot User OAuth Token in the environment that runs the sync, then run: gbrain sync --source ${sourceId}`
      : `Ask the user for their Slack app's Bot User OAuth Token and save it on stdin: gbrain connectors auth slack --token - ; then run: gbrain sync --source ${sourceId}. The token was read from GBRAIN_CONNECTOR_SLACK_TOKEN or ~/.gbrain/connectors/slack.json.`,
    'docs/guides/meeting-and-slack-sources.md#connector_auth_required');
}

export function slackChannelRefused(sourceId: string, error: SlackChannelUnavailableError): OperationError {
  return new OperationError('connector_access_denied', `Slack source ${sourceId} cannot read channel ${error.channelId}: ${error.message}. Nothing was imported.`,
    `Invite the app to that channel (type /invite @<your app> in it), or check the channel id; then run gbrain sync --source ${sourceId}.`,
    'docs/guides/meeting-and-slack-sources.md#connector_access_denied');
}

function translate(sourceId: string, cfg: SlackSourceConfig, error: unknown): unknown {
  if (error instanceof ConnectorAuthError) return slackAuthRequired(sourceId, cfg, 'Slack rejected it');
  if (error instanceof SlackChannelUnavailableError) return slackChannelRefused(sourceId, error);
  if (error instanceof ConnectorAccessDeniedError) {
    return new OperationError('connector_access_denied', `The Slack token for ${sourceId} may not do this: ${error.message}. Nothing more was imported.`,
      `Grant the app the scopes the source needs (channels:read, channels:history, groups:read, groups:history, users:read), reinstall it, save the new token with gbrain connectors auth slack --token -, then run gbrain sync --source ${sourceId}.`,
      'docs/guides/meeting-and-slack-sources.md#connector_access_denied');
  }
  if (error instanceof ConnectorRateLimitedError) {
    const at = error.retryAt ? new Date(error.retryAt).toISOString() : null;
    return new OperationError('rate_limited', `Slack rate-limited ${sourceId}${at ? ` until ${at}` : ''}; progress so far was saved.`,
      `Wait${at ? ` until ${at}` : ''}; autopilot resumes it by itself, or run gbrain sync --source ${sourceId} after that.`,
      'docs/guides/meeting-and-slack-sources.md#rate_limited');
  }
  return error;
}

export async function runSlackSync(engine: BrainEngine, sourceId: string, cfg: SlackSourceConfig, opts: SyncOpts, deps: SlackSyncDeps = {}): Promise<SyncResult> {
  return withConnectorSync(engine, sourceId, 'slack', cfg, opts,
    async (managed, options) => options.dryRun ? connectorDryRunResult('slack', sourceId) : runSlackSyncInner(engine, sourceId, cfg, options, managed, deps), pendingConnectorResult);
}

const MENTION_RE = /<@([UW][A-Z0-9]+)(?:\|[^>]*)?>/g;

async function runSlackSyncInner(engine: BrainEngine, sourceId: string, cfg: SlackSourceConfig, opts: SyncOpts,
  managed: ManagedConnectorSync | null, deps: SlackSyncDeps): Promise<SyncResult> {
  const now = deps.now ?? Date.now;
  const runStart = now();
  const runId = randomUUID();
  let api = deps.api;
  if (!api) {
    const key = resolveApiToken('slack', cfg.tokenEnv);
    if (!key) throw slackAuthRequired(sourceId, cfg);
    api = slackApi({ token: key.token, userAgent: `gbrain/${VERSION} (+https://github.com/garrytan/gbrain)`, now, maxRequests: cfg.maxRequests });
  }
  const signal = opts.signal;
  let account: SlackAccount;
  try { account = await api.authTest(signal); } catch (e) {
    if (e instanceof ConnectorRateLimitedError && e.retryAt) await recordRateLimit(engine, sourceId, e.retryAt);
    throw translate(sourceId, cfg, e);
  }
  const resolved = slackConnectorAccount(account);

  const [source] = await engine.executeRaw<{ incarnation: string; local_path: string | null; config: Record<string, unknown> }>(
    'SELECT incarnation::text AS incarnation,local_path,config FROM sources WHERE id=$1', [sourceId]);
  const identity = connectorIdentity('slack', source.config, source.local_path);
  const cursorKey = connectorDbCursorKey(sourceId, source.incarnation, identity);
  let state: SlackState;
  if (managed) {
    await managed.assertAccount(resolved);
    state = { ...emptyState(), ...managed.state<SlackState>(emptyState()) };
  } else {
    state = { ...emptyState(), ...(await readConnectorDbCursor(engine, cursorKey)).state as SlackState | null };
    const pinKey = connectorAccountPinKey(sourceId, source.incarnation, 'slack');
    const pin = (await readConnectorDbCursor(engine, pinKey)).state?.account as ConnectorAccount | undefined;
    const pinned = pin ?? state.account;
    if (pinned && !sameConnectorAccount(pinned, resolved)) throw connectorAccountChanged(sourceId, 'slack', identity.config, pinned, resolved);
    if (!pin) await writeConnectorDbCursor(engine, pinKey, { account: resolved });
    state.account = resolved;
  }

  // First sync of a channel: refuse the run, with the fix, when the app cannot read it (plan item 1).
  const infos = new Map<string, SlackChannelInfo>();
  for (const id of cfg.channels) {
    if (state.channels[id]) continue;
    try { infos.set(id, await api.conversationInfo(id, signal)); } catch (e) {
      if (e instanceof ConnectorRateLimitedError && e.retryAt) await recordRateLimit(engine, sourceId, e.retryAt);
      throw translate(sourceId, cfg, e);
    }
  }
  for (const [id, info] of infos) state.channels[id] = { committed_to: null, window: null, name: info.name, private: info.isPrivate, days: {} };

  const summary: Summary = { status: 'synced', added: 0, modified: 0, deleted: 0, chunksCreated: 0, embedded: 0, pagesAffected: [], factsRetired: 0, failedFiles: 0 };
  const holds = await ConnectorHoldSession.open(engine, sourceId, managed, emptyState(), state.item_holds, { full: opts.full });
  const newest = () => Object.values(state.channels).map(c => c.committed_to).filter((v): v is string => !!v).sort().at(-1) ?? null;
  const save = async (final: boolean) => {
    if (managed) {
      await managed.saveState(state, final && summary.status === 'synced', final ? newest() ?? undefined : undefined);
      return;
    }
    await writeConnectorDbCursor(engine, cursorKey, state, final ? { run_id: runId, finished_at: new Date(now()).toISOString(),
      counts: { added: summary.added, modified: summary.modified, deleted: summary.deleted, facts_retired: summary.factsRetired, failed: summary.failedFiles } } : undefined);
  };
  const ctx: ConnectorPublishCtx = { engine, sourceId, managed, summary, label: 'Slack', tag: 'slack', record: cfg.record, dir: cfg.dir };
  const handle = async (key: string, work: () => Promise<void>, slug: string | null): Promise<boolean> => {
    try { await work(); holds.succeed(key); return true; } catch (error) {
      if (runScoped(error)) throw error;
      summary.failedFiles++;
      await holds.fail(key, error, { slug });
      return false;
    }
  };

  let listedUsers = false;
  const ensureUsers = async (messages: SlackMessage[]) => {
    const ids = new Set<string>();
    for (const m of messages) {
      if (m.user) ids.add(m.user);
      for (const match of (m.text ?? '').matchAll(MENTION_RE)) ids.add(match[1]!);
    }
    const age = state.users_listed_at ? runStart - Date.parse(state.users_listed_at) : Infinity;
    const known = (id: string) => state.users[id] && (state.users[id]!.name !== id || !(age > USERS_MISSING_RETRY_MS));
    if (listedUsers || (!(age > USERS_MAX_AGE_MS) && [...ids].every(known))) return;
    const list = await api!.usersList(signal);
    listedUsers = true;
    state.users_listed_at = new Date(runStart).toISOString();
    for (const id of new Set([...Object.keys(state.users), ...ids])) state.users[id] = list.get(id) ?? state.users[id] ?? { name: id, deleted: false, bot: false };
  };

  const publishDay = async (info: SlackChannelInfo, day: string, messages: SlackMessage[], fp: string) => {
    const replies = new Map<string, SlackMessage[]>();
    for (const m of messages) {
      if ((m.reply_count ?? 0) > 0 && m.subtype !== 'thread_broadcast') replies.set(m.ts, await api!.replies(info.id, m.ts, signal));
    }
    await ensureUsers([...messages, ...[...replies.values()].flat()]);
    const relPath = slackPagePath(info.id, day);
    await publishConnectorPage(ctx, {
      relPath,
      markdown: connectorRender(renderSlackDay({ channel: info, day, timezone: cfg.timezone, account, messages, replies, users: state.users, privateOnly: cfg.privateOnly, skip: cfg.skip }),
        { path: relPath, slack_channel_id: info.id }),
      commitMessage: `slack(${sourceId}): #${info.name.replace(/\s+/g, ' ').slice(0, 60)} ${day}\n\nslack_channel_id: ${info.id}\nslack_team_id: ${account.teamId}`,
      privateMessage: `slack(${sourceId}): ${info.id} ${day} is private; database only`,
      retireReason: 'the Slack day changed upstream',
    });
    state.channels[info.id]!.days[day] = { path: relPath, fp };
  };

  const deleteDay = async (channelId: string, day: string) => {
    const entry = state.channels[channelId]!.days[day];
    await deleteConnectorPage(ctx, entry?.path ?? slackPagePath(channelId, day), `slack(${sourceId}): ${channelId} ${day} has no messages upstream`,
      'the Slack day was deleted upstream');
    delete state.channels[channelId]!.days[day];
  };

  /**
   * T23: pages of a private channel (or a private-only source) that are not private yet are re-stamped private from
   * their stored copy, without asking Slack: that takes them out of the checkout and retires their facts, and a day
   * Slack no longer returns (a free plan's retention) is kept, never deleted. Days inside the window re-render as usual.
   */
  const makeChannelPrivate = async (channelId: string) => {
    const rows = await engine.executeRaw<{ day: string }>(
      `SELECT DISTINCT frontmatter->>'date' AS day FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND frontmatter->>'slack_channel_id'=$2
        AND COALESCE(frontmatter->>'visibility','') <> 'private' AND frontmatter->>'date' IS NOT NULL ORDER BY 1`, [sourceId, channelId]);
    for (const { day } of rows) {
      if (!/^\d{4}-\d{2}-\d{2}$/.test(day)) continue;
      const relPath = slackPagePath(channelId, day);
      const slug = slugifyPath(relPath);
      await handle(`${channelId}/${day}`, async () => {
        const page = await engine.getPage(slug, { sourceId });
        if (!page) return;
        assertInlineBody(page, 'slack private re-stamp');
        const markdown = serializePageToMarkdown(page, await engine.getTags(slug, { sourceId }), { frontmatterOverrides: { visibility: 'private' } });
        await publishConnectorPage(ctx, { relPath, markdown: connectorRender(markdown, { path: relPath, slack_channel_id: channelId }),
          commitMessage: `slack(${sourceId}): ${channelId} ${day} is private`, privateMessage: `slack(${sourceId}): ${channelId} ${day} is private; database only`,
          retireReason: 'the Slack channel became private' });
      }, slug);
    }
  };

  /** One day read again on its own (a held day, or a day of a channel that became private). */
  const rebuildDay = async (info: SlackChannelInfo, day: string) => {
    const messages = (await api!.history(info.id, slackDayStartMs(day, cfg.timezone) / 1000, slackDayStartMs(nextSlackDay(day), cfg.timezone) / 1000, signal))
      .filter(m => !skippedMessage(m, cfg.skip));
    if (messages.length) await publishDay(info, day, messages, slackDayFingerprint({ channel: info, messages, privateOnly: cfg.privateOnly }));
    else if (state.channels[info.id]!.days[day] || await pageExists(slugifyPath(slackPagePath(info.id, day)))) await deleteDay(info.id, day);
  };

  const pageExists = async (slug: string) =>
    (await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM pages WHERE source_id=$1 AND slug=$2 AND deleted_at IS NULL', [sourceId, slug]))[0]!.n > 0;

  const syncChannel = async (channelId: string) => {
    const info = infos.get(channelId) ?? await api!.conversationInfo(channelId, signal);
    const ch = state.channels[channelId] ??= { committed_to: null, window: null, name: info.name, private: info.isPrivate, days: {} };
    Object.assign(ch, { name: info.name, private: info.isPrivate });
    if (info.isPrivate || cfg.privateOnly) await makeChannelPrivate(channelId);
    if (opts.full) Object.assign(ch, { committed_to: null, window: null });
    const window = openCoverageWindow(ch, { runStart, historyDays: cfg.historyDays, lookbackDays: cfg.lookbackDays, alignFrom: ms => slackAlignToDay(ms, cfg.timezone) });
    const from = Date.parse(window.from);
    const to = Date.parse(window.to);
    const byDay = new Map<string, SlackMessage[]>();
    for (const m of await api!.history(channelId, from / 1000, to / 1000, signal)) {
      if (skippedMessage(m, cfg.skip)) continue;
      const day = slackDayKey(Number(m.ts) * 1000, cfg.timezone);
      byDay.set(day, [...byDay.get(day) ?? [], m]);
    }
    for (const [day, messages] of [...byDay].sort(([a], [b]) => a.localeCompare(b))) {
      const key = `${channelId}/${day}`;
      if (!holds.shouldAttempt(key) || slackDayStartMs(day, cfg.timezone) < from) continue;
      const fp = slackDayFingerprint({ channel: info, messages, privateOnly: cfg.privateOnly });
      if (!opts.full && ch.days[day]?.fp === fp) continue;
      await handle(key, () => publishDay(info, day, messages, fp), slugifyPath(slackPagePath(channelId, day)));
    }
    for (const day of Object.keys(ch.days)) {
      const start = slackDayStartMs(day, cfg.timezone);
      if (byDay.has(day) || start < from || start >= to) continue;
      await handle(`${channelId}/${day}`, () => deleteDay(channelId, day), slugifyPath(ch.days[day]!.path));
    }
    // Days held by an earlier run and now outside the window are read again on their own.
    for (const key of holds.dueHeldKeys()) {
      const day = key.startsWith(`${channelId}/`) ? key.slice(channelId.length + 1) : null;
      if (!day || !/^\d{4}-\d{2}-\d{2}$/.test(day) || byDay.has(day)) continue;
      await handle(key, () => rebuildDay(info, day), slugifyPath(slackPagePath(channelId, day)));
    }
    const horizon = closeCoverageWindow(ch, cfg.lookbackDays);
    for (const day of Object.keys(ch.days)) if (slackDayStartMs(nextSlackDay(day), cfg.timezone) <= horizon) delete ch.days[day];
  };

  try {
    for (const channelId of cfg.channels) {
      if (!holds.shouldAttempt(channelId)) continue;
      await handle(channelId, () => syncChannel(channelId), null);
    }
    if (summary.failedFiles > 0) summary.status = 'partial';
    state.item_holds = holds.finish();
    await save(true);
    await holds.complete();
    if (!managed) {
      await engine.executeRaw('UPDATE sources SET last_sync_at=now(), newest_content_at=COALESCE($1::timestamptz, newest_content_at) WHERE id=$2', [newest(), sourceId]).catch(() => undefined);
    }
  } catch (error) {
    if (error instanceof ConnectorRateLimitedError && error.retryAt) await recordRateLimit(engine, sourceId, error.retryAt);
    if (runScoped(error) && !(error as { name?: string }).name?.startsWith('Lock')) {
      try { state.item_holds = holds.holds.finish().state; await save(false); } catch { /* the original error is what the caller sees */ }
    }
    // The per-run request cap (T11) ends the run partial; the open windows resume next run.
    if (!(error instanceof ConnectorRunLimitError)) throw translate(sourceId, cfg, error);
    summary.status = 'partial';
    process.stderr.write(`[slack] ${sourceId}: ${error.message}; the next sync resumes where this one stopped\n`);
  }

  await afterConnectorPublish(ctx, opts);
  return {
    status: summary.status === 'partial' ? 'partial' : summary.added + summary.modified + summary.deleted > 0 ? 'synced' : 'up_to_date',
    fromCommit: null, toCommit: '', added: summary.added, modified: summary.modified, deleted: summary.deleted, renamed: 0,
    chunksCreated: summary.chunksCreated, embedded: summary.embedded, pagesAffected: summary.pagesAffected,
    ...(summary.failedFiles > 0 ? { failedFiles: summary.failedFiles } : {}),
    ...(summary.status === 'partial' ? { filesImported: summary.added + summary.modified,
      reason: summary.failedFiles > 0 ? 'connector_item_failures' as const : signal?.aborted ? 'timeout' as const : 'connector_partial' as const } : {}),
    ...connectorHoldsResult(sourceId, holds.summary()),
  };
}