Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
39.4 KB
Raw Blame
/**
 * google-clients — fetch clients for the Gmail / Calendar / People REST APIs.
 *
 * Hand-rolled (no googleapis dep, house style — see github-source.ts):
 *  - auth via GoogleTokenProvider (vault-backed, auto-refreshing)
 *  - 401 → forceRefresh() + single retry
 *  - 403/429 → Retry-After honored (delta-seconds AND http-date). A
 *    rate-limit-class response (403 rateLimitExceeded/userRateLimitExceeded,
 *    403 quota, or 429) gets a MUCH more patient retry budget than other
 *    retryable failures — DEFAULT_RATE_LIMIT_RETRIES, exponential backoff
 *    with jitter capped at 60s — because Gmail's per-user rate limiting
 *    under backfill burst load is common and self-clearing; giving up after
 *    two tries used to poison threads that would have succeeded a few
 *    seconds later.
 *  - 403 accessNotConfigured / SERVICE_DISABLED → GoogleApiDisabledError
 *    ('api_not_enabled') naming the API, the project and the enable link
 *    Google put in the error body (the client id's project number otherwise)
 *  - 403 insufficient authentication scopes → CredentialError 'scope_missing'
 *  - uniform pageToken pagination with a safety cap
 *  - fetchImpl injectable for tests
 */

import { CredentialError } from '../creds/errors.ts';
import type { GoogleAccessProvider } from './access.ts';
import { apiEnableLink, projectNumberFromClientId } from '../creds/providers/google.ts';
import { parseRetryAfterMs } from '../github-source.ts';
import { readBoundedHttpBody } from '../guarded-http.ts';
import {
  bareAddress,
  DEFAULT_CALENDAR_ID,
  GOOGLE_SERVICE_API,
  splitAddressList,
  type GoogleService,
  type CalendarEventData,
  type ContactData,
  type GmailMessageMeta,
  type GmailThreadData,
} from './types.ts';
import { htmlToText, trimQuotedReply } from './google-render.ts';
import { truncateUtf8 } from '../text-safe.ts';
import { GMAIL_MIME_LIMITS, gmailPartHeader, inspectGmailAttachments, walkGmailMime, type GmailMimePart } from './attachment-receipts.ts';

export type FetchImpl = (url: string, init?: RequestInit) => Promise<Response>;

const GMAIL_BASE = 'https://gmail.googleapis.com/gmail/v1';
const CALENDAR_BASE = 'https://www.googleapis.com/calendar/v3';
const PEOPLE_BASE = 'https://people.googleapis.com/v1';

const PAGINATION_CAP = 500;

/** Retry budget for a 401 needing a token refresh-and-retry. */
const DEFAULT_RETRIES = 2;

/**
 * Retry budget for a rate-limit-class response (403 rateLimitExceeded /
 * userRateLimitExceeded / quota, or 429) — deliberately much larger than
 * DEFAULT_RETRIES. Gmail's per-user rate limiting under backfill burst load
 * clears on its own within seconds to low minutes; two tries (~6s total)
 * gave up before it cleared and poisoned threads that would otherwise have
 * succeeded.
 */
const DEFAULT_RATE_LIMIT_RETRIES = 6;

/** Backoff ceiling for a rate-limit retry with no Retry-After header. */
const RATE_LIMIT_BACKOFF_CAP_MS = 60_000;

/**
 * Exponential backoff with EQUAL jitter (half fixed, half random), capped at
 * RATE_LIMIT_BACKOFF_CAP_MS. Only used when Google didn't send Retry-After —
 * jitter avoids many gbrain processes hitting the same per-user/per-project
 * quota resynchronizing their retries in lockstep.
 */
export function rateLimitBackoffMs(attempt: number): number {
  const base = Math.min(RATE_LIMIT_BACKOFF_CAP_MS, 2 ** attempt * 2_000);
  const half = base / 2;
  return Math.round(half + Math.random() * half);
}

interface GoogleErrorBody {
  error?: {
    code?: number;
    status?: string;
    message?: string;
    errors?: Array<{ reason?: string }>;
    details?: Array<{ '@type'?: string; reason?: string; metadata?: Record<string, unknown>; links?: Array<{ url?: unknown }> }>;
  };
}

type ApiHint = 'gmail' | 'calendar-json' | 'people';

/** Shared request core with auth, refresh-retry, and rate-limit handling. */
export class GoogleApiClient {
  constructor(
    protected readonly tokens: GoogleAccessProvider,
    protected readonly fetchImpl: FetchImpl = fetch,
    public readonly log: (msg: string) => void = () => {},
    /** For the api_not_enabled deep link. */
    protected readonly clientId?: string,
  ) {}

  async fetchJSON<T>(
    url: string,
    apiHint: ApiHint,
    opts: { signal?: AbortSignal; retries?: number; rateLimitRetries?: number; maxResponseBytes?: number } = {},
  ): Promise<T> {
    const retries = opts.retries ?? DEFAULT_RETRIES;
    // Rate-limit-class failures get their OWN (larger) retry budget — see
    // the class doc comment. The shared `attempt` counter is bounded by
    // whichever budget is larger; each branch below still checks its own
    // budget, so a plain 401 exhausts at `retries` exactly as before.
    const rateLimitRetries = opts.rateLimitRetries ?? DEFAULT_RATE_LIMIT_RETRIES;
    const maxAttempt = Math.max(retries, rateLimitRetries);
    for (let attempt = 0; attempt <= maxAttempt; attempt++) {
      const token = await this.tokens.getAccessToken();
      const res = await this.fetchImpl(url, {
        headers: { authorization: `Bearer ${token}` },
        ...(opts.signal ? { signal: opts.signal } : {}),
      });
      const readJSON = () => opts.maxResponseBytes
        ? readBoundedHttpBody(new Response(res.body), opts.maxResponseBytes, opts.signal ?? new AbortController().signal)
          .then(bytes => JSON.parse(bytes.toString('utf8')))
        : res.json();
      if (res.ok) return (await readJSON()) as T;

      if (res.status === 401 && attempt < retries) {
        this.log('[google] HTTP 401; refreshing access token');
        await this.tokens.forceRefresh();
        continue;
      }
      const body = (await readJSON().catch(() => ({}))) as GoogleErrorBody;
      if (res.status === 403) {
        const reason = body.error?.errors?.[0]?.reason ?? '';
        const msg = body.error?.message ?? '';
        const errorInfo = body.error?.details?.find(detail => detail?.['@type'] === 'type.googleapis.com/google.rpc.ErrorInfo');
        if (reason === 'accessNotConfigured' || errorInfo?.reason === 'SERVICE_DISABLED' || /has not been used|is disabled/i.test(msg)) {
          throw googleApiDisabledError(body, apiHint, this.clientId);
        }
        if (reason === 'insufficientPermissions' || errorInfo?.reason === 'ACCESS_TOKEN_SCOPE_INSUFFICIENT' || /insufficient authentication scopes/i.test(msg)) {
          throw new CredentialError('scope_missing', undefined, `HTTP 403 on ${apiHint}: ${msg || reason}`);
        }
        if (reason === 'rateLimitExceeded' || reason === 'userRateLimitExceeded' || /rate/i.test(msg)) {
          // fall through to the retry-after sleep below
        } else if (!/quota/i.test(`${reason} ${msg}`)) {
          throw new GoogleForbiddenError(`: HTTP 403 ${reason || msg} on ${apiHint}`);
        }
      }
      if (res.status === 403 || res.status === 429) {
        const waitMs = parseRetryAfterMs(res.headers.get('retry-after')) ?? rateLimitBackoffMs(attempt);
        if (attempt < rateLimitRetries) {
          this.log(
            `[google] HTTP ${res.status}; retrying in ${Math.round(waitMs / 1000)}s ` +
              `(rate limit, attempt ${attempt + 1}/${rateLimitRetries})`,
          );
          await new Promise<void>((resolve) => {
            const t = setTimeout(resolve, waitMs);
            opts.signal?.addEventListener('abort', () => { clearTimeout(t); resolve(); }, { once: true });
          });
          continue;
        }
        throw new CredentialError('rate_limited', undefined, `HTTP ${res.status} on ${url}`);
      }
      // 404 / 410 surface to callers — cursor-expiry handling is theirs.
      if (res.status === 404 || res.status === 410) {
        throw new GoogleCursorExpiredError(res.status, url);
      }
      if (res.status === 400 && apiHint === 'people' && new URL(url).searchParams.get('syncToken')) {
        const details = body.error?.details;
        const expired = (Array.isArray(details) && details.some(detail =>
          detail?.['@type'] === 'type.googleapis.com/google.rpc.ErrorInfo' && detail.reason === 'EXPIRED_SYNC_TOKEN',
        )) || /^Sync token (?:is )?expired\b/i.test(body.error?.message ?? '');
        if (expired) throw new GoogleCursorExpiredError(res.status, url);
        throw new GoogleSyncTokenRejectedError(url, body.error?.message ?? 'no detail');
      }
      // #8906: a connections pageToken Google no longer honors (a re-list resumed from its checkpoint).
      if (res.status === 400 && apiHint === 'people') {
        const u = new URL(url);
        const pageTokenRejected = /page[ _]?token/i.test(`${body.error?.message ?? ''} ${JSON.stringify(body.error?.details ?? [])}`);
        if (u.pathname.endsWith('/people/me/connections') && u.searchParams.get('pageToken') && pageTokenRejected) throw new GoogleCursorExpiredError(res.status, url);
      }
      throw new CredentialError('upstream', `: HTTP ${res.status} on ${apiHint} (${body.error?.message ?? 'no detail'})`);
    }
    throw new CredentialError('upstream', `: unreachable ${apiHint}`);
  }

  /**
   * Drain a pageToken-paginated endpoint. `build(pageToken)` returns the URL.
   * At the page cap: throws by default (callers that RECONCILE must never
   * treat a truncated list as complete), or returns the partial batch when
   * `partialOk` is set (callers with their own resume cursor — the Gmail
   * backfill — make forward progress from whatever landed instead of
   * wedging forever on a >cap window).
   */
  async drainPages<T>(
    build: (pageToken: string | null) => string,
    pick: (body: Record<string, unknown>) => { items: T[]; nextPageToken: string | null },
    apiHint: ApiHint,
    opts: { signal?: AbortSignal; maxPages?: number; partialOk?: boolean } = {},
  ): Promise<T[]> {
    const out: T[] = [];
    let pageToken: string | null = null;
    const cap = opts.maxPages ?? PAGINATION_CAP;
    for (let page = 0; page < cap; page++) {
      const body = await this.fetchJSON<Record<string, unknown>>(build(pageToken), apiHint, opts);
      const { items, nextPageToken } = pick(body);
      out.push(...items);
      if (!nextPageToken) return out;
      pageToken = nextPageToken;
    }
    if (opts.partialOk) return out;
    throw new CredentialError('upstream', `: pagination cap (${cap}) hit on ${apiHint}`);
  }
}

const SERVICE_FOR_HINT: Record<ApiHint, GoogleService> = { gmail: 'gmail', 'calendar-json': 'calendar', people: 'contacts' };

/** Only a Google Cloud console link is relayed from an error body; anything else falls back to the derived link. */
function consoleUrl(value: unknown): string | null {
  if (typeof value !== 'string') return null;
  try {
    const url = new URL(value);
    return url.protocol === 'https:' && (url.hostname === 'console.developers.google.com' || url.hostname === 'console.cloud.google.com') ? url.toString() : null;
  } catch {
    return null;
  }
}

/**
 * #7073: an API disabled for the OAuth client's project. Carries the API, its
 * title, the project and the enable link from Google's ErrorInfo (the Help
 * link or the message when ErrorInfo is absent), so setup, status and doctor
 * can name every disabled API with its own link.
 */
export class GoogleApiDisabledError extends CredentialError {
  constructor(readonly api: string, readonly apiTitle: string, readonly project: string | null, readonly enableUrl: string) {
    super('api_not_enabled', enableUrl);
    this.name = 'GoogleApiDisabledError';
  }
}

function googleApiDisabledError(body: GoogleErrorBody, apiHint: ApiHint, clientId: string | undefined): GoogleApiDisabledError {
  const details = body.error?.details ?? [];
  const metadata = details.find(detail => detail?.['@type'] === 'type.googleapis.com/google.rpc.ErrorInfo')?.metadata ?? {};
  const text = (key: string): string | null => typeof metadata[key] === 'string' && (metadata[key] as string).trim() ? (metadata[key] as string).trim() : null;
  const helpUrl = details.flatMap(detail => detail?.links ?? []).map(link => consoleUrl(link?.url)).find(url => url !== null) ?? null;
  const messageUrl = consoleUrl(body.error?.message?.match(/https:\/\/\S+/)?.[0]?.replace(/[.,;)]+$/, ''));
  const consumer = text('consumer')?.replace(/^projects\//, '') ?? text('containerInfo');
  const project = consumer ?? body.error?.message?.match(/in project (\d+)/)?.[1] ?? (clientId ? projectNumberFromClientId(clientId) : null);
  const fallback = GOOGLE_SERVICE_API[SERVICE_FOR_HINT[apiHint]];
  return new GoogleApiDisabledError(text('service') ?? fallback.api, text('serviceTitle') ?? fallback.title, project,
    consoleUrl(metadata.activationUrl) ?? helpUrl ?? messageUrl ?? apiEnableLink(apiHint, clientId));
}

/** Gmail 404-on-historyId / Calendar-People 410-on-syncToken. */
export class GoogleCursorExpiredError extends Error {
  constructor(
    public readonly status: number,
    url: string,
  ) {
    super(`Google cursor expired (HTTP ${status}) on ${url}`);
    this.name = 'GoogleCursorExpiredError';
  }
}

/**
 * #5012 item 9: a 403 that is neither a disabled API, a rate limit nor a
 * quota: the token cannot read this resource. Still an upstream
 * `CredentialError` for every caller that does not classify it; the calendar
 * sweep and `google status` read it as an unreadable calendar.
 */
export class GoogleForbiddenError extends CredentialError {
  constructor(detail: string) {
    super('upstream', detail);
    this.name = 'GoogleForbiddenError';
  }
}

/** #5012 item 9: the HTTP status when `e` says the calendar cannot be read by this account (403 or 404), else null. */
export function calendarUnreadableStatus(e: unknown): 403 | 404 | null {
  if (e instanceof GoogleForbiddenError) return 403;
  if (e instanceof GoogleCursorExpiredError && e.status === 404) return 404;
  return null;
}

/**
 * #6324: People answered a syncToken request with a 400 that is NOT the typed
 * expiry. The People API requires every other parameter of an incremental
 * request to match the call that minted the token, so a token minted under a
 * different `personFields` lands here. Still an upstream `CredentialError`
 * for every caller that does not own a contacts cursor; the contacts sweep
 * treats it as cursor-invalid and re-lists once.
 */
export class GoogleSyncTokenRejectedError extends CredentialError {
  constructor(url: string, detail: string) {
    super('upstream', `: HTTP 400 on people (${detail})`);
    this.name = 'GoogleSyncTokenRejectedError';
    this.url = url;
  }
  readonly url: string;
}

// ── Gmail ────────────────────────────────────────────────────────────────────

type RawGmailPart = GmailMimePart;

interface RawGmailMessage {
  id: string;
  threadId: string;
  labelIds?: string[];
  internalDate?: string;
  payload?: RawGmailPart;
}

interface RawGmailThread {
  id: string;
  historyId?: string;
  messages?: RawGmailMessage[];
}

function partHeader(part: RawGmailPart | undefined, name: string): string {
  return gmailPartHeader(part, name);
}

function header(msg: RawGmailMessage, name: string): string {
  return partHeader(msg.payload, name);
}

/**
 * RFC 3834 `Auto-Submitted`: any keyword other than `no` (`auto-generated`,
 * `auto-replied`, `auto-notified`, extensions) marks machine mail, matched
 * case-insensitively with parameters and comments ignored. An absent or empty
 * header is human mail.
 */
function isAutoSubmittedHeader(value: string): boolean {
  const keyword = value.replace(/\([^)]*\)/g, ' ').split(';')[0]!.trim().toLowerCase();
  return keyword !== '' && keyword !== 'no';
}

/** iCalendar METHOD parameter of a Content-Type value (quoted or bare). */
const CALENDAR_METHOD_RE = /method\s*=\s*"?([a-z]+)"?/i;

function decodeB64Url(data: string): string {
  try {
    return Buffer.from(data.replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf-8');
  } catch {
    return '';
  }
}

/** MIME walk: prefer text/plain, fall back to text/html (caller strips). */
export function extractBody(part: RawGmailPart | undefined): { text: string; isHtml: boolean } {
  if (!part) return { text: '', isHtml: false };
  let html: string | null = null;
  const attachmentPaths: string[] = [];
  for (const { part: p, path } of walkGmailMime(part).parts) {
    if (attachmentPaths.some(parent => path.startsWith(`${parent}.`))) continue;
    if (p.filename || p.mimeType === 'message/rfc822' || /^attachment\b/i.test(partHeader(p, 'Content-Disposition'))) {
      attachmentPaths.push(path);
      continue;
    }
    if (p.mimeType === 'text/plain' && p.body?.data) {
      const plain = decodeB64Url(p.body.data);
      // A text/plain part that is itself an HTML document is markup: flag it so
      // the caller strips it instead of storing raw tags and CSS as the body.
      return { text: plain, isHtml: /^\s*(<!doctype\s|<html[\s>]|<head[\s>])/i.test(plain) };
    }
    if (p.mimeType === 'text/html' && p.body?.data && html === null) {
      html = decodeB64Url(p.body.data);
    }
  }
  if (html !== null) return { text: html, isHtml: true };
  // Single-part messages sometimes carry data at the top level with no mimeType match.
  if (!part.mimeType && !part.filename && part.body?.data) return { text: decodeB64Url(part.body.data), isHtml: false };
  return { text: '', isHtml: false };
}

/**
 * iCalendar method for a message: the METHOD of its `text/calendar` /
 * `application/ics` MIME part ('' when the part carries no parsable method),
 * or null when the message has no calendar MIME part at all — including the
 * bare-.ics-filename-attachment shape, which is a human forwarding an invite
 * file, not Calendar system mail.
 *
 * Structural, not textual: Google Calendar attaches a `text/calendar` part
 * (`method=REQUEST|REPLY|CANCEL`) to every invitation, update, response and
 * cancellation, so this identifies calendar system mail without matching on
 * subject wording or sender address — both of which are wrong signals, since
 * the mail arrives FROM the colleague's real address. Only a NON-EMPTY method
 * classifies as system mail (isCalendarSystemMail); '' and null both fall to
 * the anchored subject-prefix fallback.
 */
export function extractCalendarMethod(part: RawGmailPart | undefined): string | null {
  if (!part) return null;
  for (const { part: p } of walkGmailMime(part).parts) {
    const mime = typeof p.mimeType === 'string' ? p.mimeType.toLowerCase() : '';
    if (mime.startsWith('text/calendar') || mime === 'application/ics') {
      // Gmail's MessagePart.mimeType is the BARE media type; the `method=`
      // parameter lives in the part's own Content-Type header (format=full
      // carries headers on nested parts). Read that first — a parser that
      // only looked at mimeType read '' for every real invite, which silently
      // downgraded the structural signal to the subject-prefix fallback. A
      // mimeType that still carries the params (other providers / fixtures)
      // remains the fallback parse.
      const contentType = partHeader(p, 'Content-Type');
      const m = CALENDAR_METHOD_RE.exec(contentType) ?? CALENDAR_METHOD_RE.exec(p.mimeType ?? '');
      return (m?.[1] ?? '').toUpperCase();
    }
    // A bare `.ics` FILENAME with a non-calendar MIME type claims nothing:
    // that shape is a human attaching an invite file, not Calendar system
    // mail, and short-circuiting '' here used to suppress the whole message
    // from loop detection. Keep scanning — a real text/calendar part
    // elsewhere in the tree still wins.
  }
  return null;
}

export class GmailClient extends GoogleApiClient {
  async getProfile(opts: { signal?: AbortSignal } = {}): Promise<{ emailAddress: string; historyId: string }> {
    return this.fetchJSON(`${GMAIL_BASE}/users/me/profile`, 'gmail', opts);
  }

  /** The account's labels (system and user), for resolving `g_loops_exclude_labels` names to ids once per sweep (#5445). */
  async getLabels(opts: { signal?: AbortSignal } = {}): Promise<Array<{ id: string; name: string; type: string }>> {
    const body = await this.fetchJSON<{ labels?: Array<{ id?: string; name?: string; type?: string }> }>(`${GMAIL_BASE}/users/me/labels`, 'gmail', opts);
    return (body.labels ?? []).flatMap((l) => typeof l.id === 'string' && typeof l.name === 'string' ? [{ id: l.id, name: l.name, type: l.type ?? 'user' }] : []);
  }

  /** Message ids matching a Gmail search query (includes SENT; excludes SPAM/TRASH). */
  async listMessageIds(
    q: string,
    opts: { signal?: AbortSignal; maxPages?: number; partialOk?: boolean } = {},
  ): Promise<Array<{ id: string; threadId: string }>> {
    return this.drainPages(
      (t) =>
        `${GMAIL_BASE}/users/me/messages?maxResults=100&q=${encodeURIComponent(q)}${t ? `&pageToken=${encodeURIComponent(t)}` : ''}`,
      (body) => ({
        items: (body.messages as Array<{ id: string; threadId: string }> | undefined) ?? [],
        nextPageToken: (body.nextPageToken as string | undefined) ?? null,
      }),
      'gmail',
      opts,
    );
  }

  /**
   * Thread ids touched since the stored historyId. Throws
   * GoogleCursorExpiredError(404) when the cursor is too old (~1 week).
   * Returns the new historyId to store after a successful drain.
   *
   * #5581 (fix wave 4): with `maxThreads`, the listing stops before the
   * history record that would exceed it and returns `truncated: true` with
   * `newHistoryId` set to the last record fully included, so the caller can
   * bank that far and resume from there without dropping any change. A
   * record without an id cannot be bounded safely, so the listing then
   * drains everything.
   */
  async listHistoryThreadIds(
    startHistoryId: string,
    opts: { signal?: AbortSignal; maxThreads?: number } = {},
  ): Promise<{ threadIds: string[]; newHistoryId: string | null; truncated: boolean; versions: Map<string, string> }> {
    const threadIds = new Set<string>();
    // The latest history record touching each thread: its upstream version for the item holds.
    const versions = new Map<string, string>();
    let newHistoryId: string | null = null;
    let lastRecordId: string | null = null;
    let truncated = false;
    await this.drainPages(
      (t) =>
        `${GMAIL_BASE}/users/me/history?startHistoryId=${encodeURIComponent(startHistoryId)}&maxResults=100${t ? `&pageToken=${encodeURIComponent(t)}` : ''}`,
      (body) => {
        const records = (body.history as Array<Record<string, unknown>> | undefined) ?? [];
        for (const rec of records) {
          const touched = new Set<string>();
          for (const key of ['messages', 'messagesAdded', 'messagesDeleted', 'labelsAdded', 'labelsRemoved']) {
            const arr = rec[key] as Array<{ threadId?: string; message?: { threadId?: string } }> | undefined;
            for (const m of arr ?? []) {
              const tid = m.threadId ?? m.message?.threadId;
              if (tid) touched.add(tid);
            }
          }
          const fresh = [...touched].filter((tid) => !threadIds.has(tid));
          const recordId = typeof rec.id === 'string' ? rec.id : null;
          if (opts.maxThreads !== undefined && lastRecordId && recordId && threadIds.size + fresh.length > opts.maxThreads) {
            truncated = true;
            return { items: [], nextPageToken: null };
          }
          for (const tid of fresh) threadIds.add(tid);
          if (recordId) {
            lastRecordId = recordId;
            for (const tid of touched) versions.set(tid, recordId);
          }
        }
        if (typeof body.historyId === 'string') newHistoryId = body.historyId;
        return { items: [], nextPageToken: (body.nextPageToken as string | undefined) ?? null };
      },
      'gmail',
      opts,
    );
    return { threadIds: [...threadIds], newHistoryId: truncated ? lastRecordId : newHistoryId, truncated, versions };
  }

  async getThread(
    threadId: string,
    account: string,
    opts: { signal?: AbortSignal; bodyCapChars?: number; metadataOnly?: boolean } = {},
  ): Promise<GmailThreadData> {
    let fields = '';
    if (opts.metadataOnly) {
      let payload = 'partId,mimeType';
      for (let depth = GMAIL_MIME_LIMITS.depth; depth >= 0; depth--) {
        payload = `partId,mimeType,filename,headers(name,value),body(attachmentId,size),parts(${payload})`;
      }
      fields = `&fields=${encodeURIComponent(`id,messages(id,internalDate,labelIds,payload(${payload}))`)}`;
    }
    const raw = await this.fetchJSON<RawGmailThread>(
      `${GMAIL_BASE}/users/me/threads/${encodeURIComponent(threadId)}?format=full${fields}`,
      'gmail',
      opts.metadataOnly ? { ...opts, maxResponseBytes: 2 * 1024 * 1024 } : opts,
    );
    const cap = opts.bodyCapChars ?? 8_000;
    if (opts.metadataOnly && (typeof raw.id !== 'string' || !raw.id || !Array.isArray(raw.messages) || !raw.messages.length ||
      raw.messages.some(m => !m || typeof m.id !== 'string' || !/^[A-Za-z0-9]{1,128}$/.test(m.id)) ||
      new Set(raw.messages.map(m => m.id)).size !== raw.messages.length)) {
      throw new Error('Gmail returned malformed thread identities; historical metadata was preserved.');
    }
    const normalizedThreadId = typeof raw.id === 'string' && raw.id ? raw.id : threadId;
    let receiptBudget: number = GMAIL_MIME_LIMITS.receiptBytes;
    const messages: GmailMessageMeta[] = (raw.messages ?? []).map((m, index) => {
      const messageId = typeof m.id === 'string' && m.id ? m.id : `missing:${threadId}:${index}`;
      const fromRaw = header(m, 'From');
      const { text: rawText, isHtml } = opts.metadataOnly ? { text: '', isHtml: false } : extractBody(m.payload);
      // Pre-truncate before conversion: only the first `cap` output chars
      // survive, so a multi-hundred-KB marketing email must not pay ~15
      // full-body regex passes in htmlToText inside the per-thread hot loop.
      const text = truncateUtf8(rawText, cap * 16);
      let bodyText = isHtml ? htmlToText(text) : text;
      bodyText = trimQuotedReply(bodyText);
      if (bodyText.length > cap) bodyText = truncateUtf8(bodyText, cap) + '\n[truncated]';
      const internalDateMs = Number(m.internalDate ?? 0);
      const attachmentInspection = inspectGmailAttachments(m.payload, account, messageId, receiptBudget);
      receiptBudget -= Buffer.byteLength(JSON.stringify(attachmentInspection));
      return {
        id: messageId,
        threadId: normalizedThreadId,
        from: fromRaw,
        fromAddress: bareAddress(fromRaw),
        to: splitAddressList(header(m, 'To')),
        cc: splitAddressList(header(m, 'Cc')),
        subject: header(m, 'Subject'),
        dateIso: internalDateMs > 0 ? new Date(internalDateMs).toISOString() : new Date(0).toISOString(),
        internalDateMs,
        labelIds: m.labelIds ?? [],
        listUnsubscribe: header(m, 'List-Unsubscribe') !== '',
        autoSubmitted: isAutoSubmittedHeader(header(m, 'Auto-Submitted')),
        precedence: header(m, 'Precedence').trim().toLowerCase(),
        listId: header(m, 'List-Id') !== '',
        replyTo: splitAddressList(header(m, 'Reply-To')),
        calendarMethod: extractCalendarMethod(m.payload),
        bodyText,
        attachmentInspection,
      };
    });
    messages.sort((a, b) => a.internalDateMs - b.internalDateMs);
    return { threadId: normalizedThreadId, account, messages };
  }
}

// ── Calendar ─────────────────────────────────────────────────────────────────

interface RawCalendarEvent {
  id: string;
  etag?: string;
  status?: string;
  summary?: string;
  description?: string;
  start?: { dateTime?: string; date?: string };
  end?: { dateTime?: string; date?: string };
  organizer?: { email?: string };
  attendees?: Array<{ email?: string; displayName?: string; self?: boolean; responseStatus?: string }>;
  location?: string;
  hangoutLink?: string;
  htmlLink?: string;
  visibility?: string;
}

function calendarEvent(e: RawCalendarEvent, account: string): CalendarEventData {
  return {
    id: e.id,
    ...(e.etag ? { etag: e.etag } : {}),
    summary: e.summary ?? '(no title)',
    description: e.description ?? '',
    startIso: e.start?.dateTime ?? (e.start?.date ? `${e.start.date}T00:00:00Z` : ''),
    endIso: e.end?.dateTime ?? (e.end?.date ? `${e.end.date}T00:00:00Z` : ''),
    allDay: Boolean(e.start?.date),
    organizer: e.organizer?.email?.toLowerCase() ?? null,
    attendees: (e.attendees ?? []).map((a) => ({
      email: (a.email ?? '').toLowerCase(),
      displayName: a.displayName ?? null,
      self: a.self ?? false,
      responseStatus: a.responseStatus ?? null,
    })),
    location: e.location ?? null,
    hangoutLink: e.hangoutLink ?? null,
    htmlLink: e.htmlLink ?? null,
    status: e.status ?? 'confirmed',
    account,
    private: e.visibility === 'private' || e.visibility === 'confidential',
  };
}

export class CalendarClient extends GoogleApiClient {
  /**
   * Incremental when syncToken is set; windowed otherwise. Throws
   * GoogleCursorExpiredError(410) on an expired syncToken — caller drops the
   * token and re-runs windowed.
   */
  async listEvents(
    account: string,
    opts: {
      syncToken?: string | null;
      timeMinIso?: string;
      timeMaxIso?: string;
      signal?: AbortSignal;
      /** Calendar to sweep. Defaults to DEFAULT_CALENDAR_ID. A secondary calendar id is
       *  an address like `[email protected]`; each calendar gets
       *  its OWN gbrain source so their sync tokens never collide. */
      calendarId?: string;
    },
  ): Promise<{ events: CalendarEventData[]; nextSyncToken: string | null }> {
    let nextSyncToken: string | null = null;
    const calId = encodeURIComponent(opts.calendarId?.trim() || DEFAULT_CALENDAR_ID);
    const base = `${CALENDAR_BASE}/calendars/${calId}/events?maxResults=250&singleEvents=true`;
    const raw = await this.drainPages<RawCalendarEvent>(
      (t) => {
        const params = new URLSearchParams();
        if (opts.syncToken) params.set('syncToken', opts.syncToken);
        else {
          if (opts.timeMinIso) params.set('timeMin', opts.timeMinIso);
          if (opts.timeMaxIso) params.set('timeMax', opts.timeMaxIso);
        }
        if (t) params.set('pageToken', t);
        const qs = params.toString();
        return qs ? `${base}&${qs}` : base;
      },
      (body) => {
        if (typeof body.nextSyncToken === 'string') nextSyncToken = body.nextSyncToken;
        return {
          items: (body.items as RawCalendarEvent[] | undefined) ?? [],
          nextPageToken: (body.nextPageToken as string | undefined) ?? null,
        };
      },
      'calendar-json',
      opts,
    );
    return { events: raw.map((e) => calendarEvent(e, account)), nextSyncToken };
  }

  /**
   * GBRA-130 A-3: one event by id, for re-attempting a held event that no
   * listing of this run returns. Null when the event is gone (404/410).
   */
  async getEvent(account: string, calendarId: string, eventId: string, opts: { signal?: AbortSignal } = {}): Promise<CalendarEventData | null> {
    const calId = encodeURIComponent(calendarId.trim() || DEFAULT_CALENDAR_ID);
    try {
      return calendarEvent(await this.fetchJSON<RawCalendarEvent>(`${CALENDAR_BASE}/calendars/${calId}/events/${encodeURIComponent(eventId)}`, 'calendar-json', opts), account);
    } catch (e) {
      if (e instanceof GoogleCursorExpiredError) return null;
      throw e;
    }
  }

  /**
   * Enumerate every calendar the account can read (calendarList). Discovery
   * only — the sweep still reads ONE calendar per source, so this exists to
   * find the id you pass to `sources add --calendar-id`.
   */
  /**
   * #5012 item 9: one cheap read of the calendar's events (`maxResults=1`).
   * `readable: false` carries the 403/404 that says the account cannot read
   * it; any other failure throws.
   */
  async probeCalendar(calendarId: string, opts: { signal?: AbortSignal } = {}): Promise<{ readable: true } | { readable: false; httpStatus: 403 | 404 }> {
    try {
      await this.fetchJSON(`${CALENDAR_BASE}/calendars/${encodeURIComponent(calendarId.trim() || DEFAULT_CALENDAR_ID)}/events?maxResults=1&fields=kind`,
        'calendar-json', { ...opts, retries: 1, rateLimitRetries: 1 });
      return { readable: true };
    } catch (e) {
      const httpStatus = calendarUnreadableStatus(e);
      if (httpStatus === null) throw e;
      return { readable: false, httpStatus };
    }
  }

  async listCalendars(opts: { signal?: AbortSignal } = {}): Promise<
    Array<{ id: string; summary: string; primary: boolean; accessRole: string }>
  > {
    const base = `${CALENDAR_BASE}/users/me/calendarList?maxResults=250`;
    const raw = await this.drainPages<{
      id?: string;
      summary?: string;
      primary?: boolean;
      accessRole?: string;
    }>(
      (t) => (t ? `${base}&pageToken=${encodeURIComponent(t)}` : base),
      (body) => ({
        items: (body.items as Array<Record<string, unknown>> | undefined) ?? [],
        nextPageToken: (body.nextPageToken as string | undefined) ?? null,
      }),
      'calendar-json',
      opts,
    );
    return raw
      .filter((c) => typeof c.id === 'string' && c.id.length > 0)
      .map((c) => ({
        id: c.id as string,
        summary: c.summary ?? '(no name)',
        primary: Boolean(c.primary),
        accessRole: c.accessRole ?? 'unknown',
      }));
  }
}

// ── People (Contacts) ────────────────────────────────────────────────────────

interface RawPerson {
  resourceName: string;
  etag?: string;
  names?: Array<{ displayName?: string; metadata?: { primary?: boolean } }>;
  emailAddresses?: Array<{ value?: string }>;
  phoneNumbers?: Array<{ value?: string; canonicalForm?: string }>;
  organizations?: Array<{ name?: string; title?: string; metadata?: { primary?: boolean } }>;
  memberships?: Array<{ contactGroupMembership?: { contactGroupResourceName?: string } }>;
  metadata?: { deleted?: boolean };
}

interface RawContactGroup {
  resourceName?: string;
  name?: string;
  formattedName?: string;
  groupType?: string;
}

/**
 * #6324: the People field mask every contacts request carries. A syncToken is
 * only valid for requests with the parameters that minted it, so the sweep
 * records this string next to the token (`contacts_person_fields`) and
 * re-lists when it changes. Change it here and nowhere else.
 */
export const CONTACT_PERSON_FIELDS = 'names,emailAddresses,organizations,phoneNumbers,memberships';

/** Google's built-in groups (`contactGroups/myContacts`, `starred`, …) say nothing a page should carry. */
const SYSTEM_CONTACT_GROUPS = new Set(
  ['all', 'myContacts', 'starred', 'blocked', 'chatBuddies', 'friends', 'family', 'coworkers'].map((id) => `contactGroups/${id}`),
);

/** Bounded directory walk: 1000 groups a page, ten pages; past that the result is explicitly incomplete. */
const CONTACT_GROUP_PAGES = 10;

function contactData(p: RawPerson): ContactData {
  const primaryName = p.names?.find((n) => n.metadata?.primary) ?? p.names?.[0];
  const primaryOrg = p.organizations?.find((o) => o.metadata?.primary) ?? p.organizations?.[0];
  return {
    resourceName: p.resourceName,
    ...(p.etag ? { etag: p.etag } : {}),
    displayName: primaryName?.displayName ?? null,
    emails: (p.emailAddresses ?? [])
      .map((e) => (e.value ?? '').trim().toLowerCase())
      .filter((e) => e.includes('@')),
    phones: [...new Set((p.phoneNumbers ?? [])
      .map((n) => (n.canonicalForm ?? n.value ?? '').trim())
      .filter((n) => n.length > 0))],
    organization: primaryOrg?.name ?? null,
    title: primaryOrg?.title ?? null,
    groups: [...new Set((p.memberships ?? [])
      .map((m) => m.contactGroupMembership?.contactGroupResourceName ?? '')
      .filter((g) => g.startsWith('contactGroups/') && !SYSTEM_CONTACT_GROUPS.has(g)))]
      .map((id) => ({ id, name: null })),
    deleted: p.metadata?.deleted ?? false,
  };
}

function connectionsUrl(syncToken: string | null | undefined, pageToken: string | null): string {
  const params = new URLSearchParams({ personFields: CONTACT_PERSON_FIELDS, pageSize: '200', requestSyncToken: 'true' });
  if (syncToken) params.set('syncToken', syncToken);
  if (pageToken) params.set('pageToken', pageToken);
  return `${PEOPLE_BASE}/people/me/connections?${params.toString()}`;
}

export class PeopleClient extends GoogleApiClient {
  /** Incremental with syncToken; full otherwise. 410 → GoogleCursorExpiredError. */
  async listConnections(opts: {
    syncToken?: string | null;
    signal?: AbortSignal;
  }): Promise<{ contacts: ContactData[]; nextSyncToken: string | null }> {
    let nextSyncToken: string | null = null;
    const raw = await this.drainPages<RawPerson>(
      (t) => connectionsUrl(opts.syncToken, t),
      (body) => {
        if (typeof body.nextSyncToken === 'string') nextSyncToken = body.nextSyncToken;
        return {
          items: (body.connections as RawPerson[] | undefined) ?? [],
          nextPageToken: (body.nextPageToken as string | undefined) ?? null,
        };
      },
      'people',
      opts,
    );
    return { contacts: raw.map(contactData), nextSyncToken };
  }

  /**
   * #8906: one page of a full list, so a re-list can checkpoint between pages.
   * `nextSyncToken` arrives only on the last page; a stale `pageToken` (400 or
   * 410) is a GoogleCursorExpiredError.
   */
  async listConnectionsPage(pageToken: string | null, opts: { signal?: AbortSignal } = {}):
    Promise<{ contacts: ContactData[]; nextPageToken: string | null; nextSyncToken: string | null }> {
    const body = await this.fetchJSON<Record<string, unknown>>(connectionsUrl(null, pageToken), 'people', opts);
    return {
      contacts: ((body.connections as RawPerson[] | undefined) ?? []).map(contactData),
      nextPageToken: (body.nextPageToken as string | undefined) ?? null,
      nextSyncToken: typeof body.nextSyncToken === 'string' ? body.nextSyncToken : null,
    };
  }

  /**
   * GBRA-130 A-3: one contact by resource name, for re-attempting a held
   * contact that no listing of this run returns. Null when it is gone (404/410).
   */
  async getContact(resourceName: string, opts: { signal?: AbortSignal } = {}): Promise<ContactData | null> {
    const path = resourceName.split('/').map(encodeURIComponent).join('/');
    try {
      return contactData(await this.fetchJSON<RawPerson>(`${PEOPLE_BASE}/${path}?personFields=${CONTACT_PERSON_FIELDS}`, 'people', opts));
    } catch (e) {
      if (e instanceof GoogleCursorExpiredError) return null;
      throw e;
    }
  }

  /**
   * #6324: the account's user-created contact groups, resource name → display
   * name. `complete` is false when the directory has more pages than the
   * bounded walk read; the caller keeps raw ids for whatever is missing
   * instead of treating one page as the whole directory.
   */
  async listContactGroups(opts: { signal?: AbortSignal } = {}): Promise<{ names: Record<string, string>; complete: boolean }> {
    let morePages = false;
    const raw = await this.drainPages<RawContactGroup>(
      (t) => {
        const params = new URLSearchParams({ pageSize: '1000', groupFields: 'name,groupType' });
        if (t) params.set('pageToken', t);
        return `${PEOPLE_BASE}/contactGroups?${params.toString()}`;
      },
      (body) => {
        const nextPageToken = (body.nextPageToken as string | undefined) ?? null;
        morePages = nextPageToken !== null;
        return { items: (body.contactGroups as RawContactGroup[] | undefined) ?? [], nextPageToken };
      },
      'people',
      { ...opts, maxPages: CONTACT_GROUP_PAGES, partialOk: true },
    );
    const names: Record<string, string> = {};
    for (const g of raw) {
      if (!g.resourceName || g.groupType === 'SYSTEM_CONTACT_GROUP' || SYSTEM_CONTACT_GROUPS.has(g.resourceName)) continue;
      const name = (g.formattedName ?? g.name ?? '').trim();
      if (name) names[g.resourceName] = name;
    }
    return { names, complete: !morePages };
  }
}