Multiplayer agent harness for work.
Share one Gmail REST client across sent mail, push, inbox refresh and send (#2492)
Regan Bell76ff0cd38846
diff --git a/src/api/routes/connectors.ts b/src/api/routes/connectors.tsindex 438693296..d3b2b5c3e 100644--- a/src/api/routes/connectors.ts+++ b/src/api/routes/connectors.ts@@ -1,4 +1,5 @@-import { GmailReadError, getSentEmail, listSentEmails } from "../../connectors/gmail-sent.ts";+import { getSentEmail, listSentEmails } from "../../connectors/gmail-sent.ts";+import { GmailApiError } from "../../connectors/gmail-api.ts";import { orgId as configOrgId } from "../../config.ts";import {authorizeUrl,@@ -534,14 +535,13 @@ export async function gmailSent(ctx: ApiCtx): Promise<void> {accountType,});} catch (error) {- let status = 502;- if (error instanceof GmailReadError) {- if ([401, 403].includes(error.status)) status = 409;- else if (error.status === 404) status = 404;- }+ const upstream = error instanceof GmailApiError ? error.status : 0;+ const reconnect = upstream === 401 || upstream === 403;+ let status = upstream === 404 ? 404 : 502;+ if (reconnect) status = 409;return sendJson(ctx.res, status, {error: "gmail_unavailable",- message: error instanceof GmailReadError ? error.message : "Couldn't load sent mail from Gmail. Try again.",+ message: reconnect ? "Reconnect Google to read sent mail." : "Couldn't load sent mail from Gmail. Try again.",});}}diff --git a/src/connectors/gmail-api.ts b/src/connectors/gmail-api.tsnew file mode 100644index 000000000..99144f042--- /dev/null+++ b/src/connectors/gmail-api.ts@@ -0,0 +1,34 @@+export class GmailApiError extends Error {+ readonly status: number;+ readonly retryAfterSeconds: number;+ readonly detail: string;++ constructor(path: string, status: number, retryAfterSeconds: number, detail: string) {+ super(`Gmail ${path.split("?")[0]} returned ${status}`);+ this.status = status;+ this.retryAfterSeconds = retryAfterSeconds;+ this.detail = detail;+ }+}++export function gmailApi(token: string, fetchImpl: typeof fetch = fetch) {+ return async <T>(path: string, { body, timeoutMs = 20_000 }: { body?: unknown; timeoutMs?: number } = {}) => {+ const response = await fetchImpl(`https://gmail.googleapis.com/gmail/v1/users/me/${path}`, {+ method: body === undefined ? "GET" : "POST",+ headers: {+ authorization: `Bearer ${token}`,+ ...(body === undefined ? {} : { "content-type": "application/json" }),+ },+ ...(body === undefined ? {} : { body: JSON.stringify(body) }),+ signal: AbortSignal.timeout(timeoutMs),+ });+ if (!response.ok)+ throw new GmailApiError(+ path,+ response.status,+ Number(response.headers.get("retry-after")),+ (await response.text().catch(() => "")).slice(0, 300),+ );+ return (await response.json()) as T;+ };+}diff --git a/src/connectors/gmail-sent.ts b/src/connectors/gmail-sent.tsindex 212993175..d1dd41cbe 100644--- a/src/connectors/gmail-sent.ts+++ b/src/connectors/gmail-sent.ts@@ -1,3 +1,5 @@+import { GmailApiError, gmailApi } from "./gmail-api.ts";+export interface SentEmail {id: string;threadId: string;@@ -24,32 +26,13 @@ interface GmailMessage {payload?: GmailPart;}-export class GmailReadError extends Error {- readonly status: number;-- constructor(status: number) {- super(- status === 401 || status === 403- ? "Reconnect Google to read sent mail."- : "Couldn't load sent mail from Gmail. Try again.",- );- this.status = status;- }-}-export async function listSentEmails(token: string,pageToken?: string,fetchImpl: typeof fetch = fetch,): Promise<{ messages: SentEmail[]; nextPageToken?: string; accountEmail: string }> {- const get = async <T>(path: string): Promise<T> => {- const response = await fetchImpl(`https://gmail.googleapis.com/gmail/v1/users/me/${path}`, {- headers: { authorization: `Bearer ${token}` },- signal: AbortSignal.timeout(15000),- });- if (!response.ok) throw new GmailReadError(response.status);- return response.json() as Promise<T>;- };+ const api = gmailApi(token, fetchImpl);+ const get = <T>(path: string): Promise<T> => api<T>(path, { timeoutMs: 15_000 });const params = new URLSearchParams({ labelIds: "SENT", maxResults: "25" });if (pageToken) params.set("pageToken", pageToken);const page = await get<{ messages?: { id: string }[]; nextPageToken?: string }>(`messages?${params}`);@@ -65,7 +48,7 @@ export async function listSentEmails(`messages/${encodeURIComponent(id)}?format=metadata&metadataHeaders=To&metadataHeaders=Subject`,);} catch (error) {- if (error instanceof GmailReadError && error.status === 404) return null;+ if (error instanceof GmailApiError && error.status === 404) return null;throw error;}const header = (name: string): string =>@@ -97,16 +80,9 @@ export async function getSentEmail(): Promise<SentEmail & { from: string; cc: string; body: string; rfcMessageId: string; html: boolean; attachments: string[] }> {- const response = await fetchImpl(- `https://gmail.googleapis.com/gmail/v1/users/me/messages/${encodeURIComponent(id)}?format=full`,- {- headers: { authorization: `Bearer ${token}` },- signal: AbortSignal.timeout(15000),- },- );- if (!response.ok) throw new GmailReadError(response.status);- const message = (await response.json()) as GmailMessage;- if (!message.labelIds?.includes("SENT")) throw new GmailReadError(404);+ const path = `messages/${encodeURIComponent(id)}?format=full`;+ const message = await gmailApi(token, fetchImpl)<GmailMessage>(path, { timeoutMs: 15_000 });+ if (!message.labelIds?.includes("SENT")) throw new GmailApiError(path, 404, 0, "");const header = (name: string): string =>message.payload?.headers?.find((h) => h.name.toLowerCase() === name)?.value ?? "";const parts: GmailPart[] = [];diff --git a/src/loops/gmail-push.ts b/src/loops/gmail-push.tsindex 3a21974f8..a84ca567e 100644--- a/src/loops/gmail-push.ts+++ b/src/loops/gmail-push.ts@@ -1,6 +1,7 @@import { createRemoteJWKSet, jwtVerify } from "jose";import type { ConnectorTokenSource } from "./sources/adapter.ts";import type { IngestEntryInput } from "./item-ledger.ts";+import { GmailApiError, gmailApi } from "../connectors/gmail-api.ts";export interface GmailPushConfig {topic: string;@@ -40,29 +41,18 @@ interface GmailMessage {}export function createGmailPushClient(tokens: ConnectorTokenSource, config: GmailPushConfig, fetchImpl = fetch) {- async function request<T>(owner: string, path: string, body?: unknown): Promise<T> {+ async function client(owner: string) {const token = await tokens.connectorAccessToken("gmail.googleapis.com", owner, "personal");if (!token) throw new Error("Connect a personal Gmail account before enabling Gmail Pub/Sub.");- const response = await fetchImpl(`https://gmail.googleapis.com/gmail/v1/users/me/${path}`, {- method: body === undefined ? "GET" : "POST",- headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },- ...(body === undefined ? {} : { body: JSON.stringify(body) }),- signal: AbortSignal.timeout(20_000),- });- if (!response.ok)- throw Object.assign(new Error(`Gmail ${path.split("?")[0]} returned ${response.status}`), {- status: response.status,- });- return (await response.json()) as T;+ return gmailApi(token, fetchImpl);}return {async watch(owner: string): Promise<GmailCursor> {- const profile = await request<{ emailAddress: string }>(owner, "profile");- const watch = await request<{ historyId: string; expiration: string }>(owner, "watch", {- topicName: config.topic,- labelIds: ["INBOX"],- labelFilterBehavior: "include",+ const request = await client(owner);+ const profile = await request<{ emailAddress: string }>("profile");+ const watch = await request<{ historyId: string; expiration: string }>("watch", {+ body: { topicName: config.topic, labelIds: ["INBOX"], labelFilterBehavior: "include" },});if (!profile.emailAddress || !/^\d+$/.test(watch.historyId) || !Number.isFinite(Number(watch.expiration)))throw new Error("Gmail returned an invalid watch response");@@ -78,16 +68,17 @@ export function createGmailPushClient(tokens: ConnectorTokenSource, config: Gmaicursor: GmailCursor,ingest: (entries: IngestEntryInput[]) => Promise<unknown>,): Promise<string> {- const profile = await request<{ emailAddress: string; historyId: string }>(owner, "profile");+ const request = await client(owner);+ const profile = await request<{ emailAddress: string; historyId: string }>("profile");if (profile.emailAddress.toLowerCase() !== cursor.email)throw new Error("Connected Gmail account changed; reconnect this ingestion source.");const importMessages = async (ids: string[]) => {for (const id of new Set(ids)) {let message: GmailMessage;try {- message = await request<GmailMessage>(owner, `messages/${encodeURIComponent(id)}?format=metadata`);+ message = await request<GmailMessage>(`messages/${encodeURIComponent(id)}?format=metadata`);} catch (error) {- if ((error as { status?: number }).status === 404) continue;+ if (error instanceof GmailApiError && error.status === 404) continue;throw error;}if (!message.labelIds?.includes("INBOX")) continue;@@ -136,7 +127,7 @@ export function createGmailPushClient(tokens: ConnectorTokenSource, config: GmaimessagesAdded?: Array<{ message: GmailMessage }>;labelsAdded?: Array<{ message: GmailMessage; labelIds: string[] }>;}>;- }>(owner, `history?${query}`);+ }>(`history?${query}`);await importMessages((page.history ?? []).flatMap((entry) => [...(entry.messagesAdded ?? []).map((row) => row.message.id),@@ -147,15 +138,12 @@ export function createGmailPushClient(tokens: ConnectorTokenSource, config: Gmailatest = page.historyId;} while (pageToken);} catch (error) {- if ((error as { status?: number }).status !== 404) throw error;+ if (!(error instanceof GmailApiError && error.status === 404)) throw error;pageToken = undefined;do {const query = new URLSearchParams({ labelIds: "INBOX", maxResults: "100" });if (pageToken) query.set("pageToken", pageToken);- const page = await request<{ nextPageToken?: string; messages?: Array<{ id: string }> }>(- owner,- `messages?${query}`,- );+ const page = await request<{ nextPageToken?: string; messages?: Array<{ id: string }> }>(`messages?${query}`);await importMessages((page.messages ?? []).map((message) => message.id));pageToken = page.nextPageToken;} while (pageToken);diff --git a/src/loops/inbox-source-refresh.ts b/src/loops/inbox-source-refresh.tsindex eaeb5ef0a..4bb3951e9 100644--- a/src/loops/inbox-source-refresh.ts+++ b/src/loops/inbox-source-refresh.ts@@ -8,6 +8,7 @@ import type { LoopItemLedger } from "./item-ledger.ts";import { isResolved } from "./ledger-view.ts";import { isObj, tokenFor, type ConnectorTokenSource } from "./sources/adapter.ts";import { normalizeSlackApiUrl } from "../slack/config.ts";+import { GmailApiError, gmailApi } from "../connectors/gmail-api.ts";import type { DirectoryStore } from "../directory/directory-store.ts";interface RefreshDeps {@@ -61,26 +62,33 @@ export function createInboxSourceRefresh(deps: RefreshDeps): InboxSourceRefreshconst method = path.split("?")[0]!;const key = `${owner}:${source}:${source === "slack" ? method : "read"}`;if (limited.has(key)) throw new Error("Source rate limited. The next refresh will retry.");- const base = source === "slack" ? slackBase : "https://gmail.googleapis.com/gmail/v1/users/me/";- const res = await fetchImpl(`${base}${path}`, {- headers: { authorization: `Bearer ${token}` },- signal: AbortSignal.timeout(Math.max(1, Math.min(3000, deadline - now()))),- });- if (res.status === 429) {- const retry = Number(res.headers.get("retry-after"));+ const timeoutMs = Math.max(1, Math.min(3000, deadline - now()));+ const rateLimit = (retry: number) =>limited.set(key, true, { ttl: Math.max(60, Number.isFinite(retry) ? retry : 60) * 1000 });+ const failure = (cause: Error) =>+ new Error("Couldn't refresh the source. Your item has not been dismissed.", { cause });+ if (source === "gmail") {+ try {+ return await gmailApi(token, fetchImpl)<Record<string, unknown>>(path, { timeoutMs });+ } catch (error) {+ if (!(error instanceof GmailApiError)) throw error;+ if (error.status === 429) rateLimit(error.retryAfterSeconds);+ throw failure(error);+ }}- if (!res.ok)- throw new Error("Couldn't refresh the source. Your item has not been dismissed.", {- cause: new Error(`${source} ${method}: HTTP ${res.status}`),- });+ const res = await fetchImpl(`${slackBase}${path}`, {+ headers: { authorization: `Bearer ${token}` },+ signal: AbortSignal.timeout(timeoutMs),+ });+ if (res.status === 429) rateLimit(Number(res.headers.get("retry-after")));+ if (!res.ok) throw failure(new Error(`${source} ${method}: HTTP ${res.status}`));const body: unknown = await res.json();if (!isObj(body) || body.ok === false)- throw new Error("Couldn't refresh the source. Your item has not been dismissed.", {- cause: new Error(+ throw failure(+ new Error(`${source} ${method}: ${isObj(body) && typeof body.error === "string" ? body.error : "Invalid response"}`,),- });+ );return body;}diff --git a/src/loops/sources/gmail.ts b/src/loops/sources/gmail.tsindex 7f4192236..314a987cb 100644--- a/src/loops/sources/gmail.ts+++ b/src/loops/sources/gmail.ts@@ -1,6 +1,7 @@import MailComposer from "nodemailer/lib/mail-composer";import type { LoopItem, LoopSourcePayload } from "../../types.ts";import { isAccountType } from "../../connectors/oauth.ts";+import { GmailApiError, gmailApi } from "../../connectors/gmail-api.ts";import { errMessage } from "../../util/errors.ts";import {addressList,@@ -101,18 +102,17 @@ async function sendGmail(deps: SourceActionDeps, item: LoopItem, draft: ReplyDraconst mime = await buildGmailReplyMime(item, draft);if (!mime) return { ok: false, reason: "bad_item", message: "no recipient — add a To: address to the draft" };const threadId = metaOf(item)?.threadId;- const res = await fetchImpl(`https://${GMAIL_HOST}/gmail/v1/users/me/messages/send`, {- method: "POST",- signal: AbortSignal.timeout(30000),- headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },- body: JSON.stringify({- raw: mime.toString("base64url"),- ...(threadId ? { threadId } : {}),- }),- });- if (!res.ok) {- const text = (await res.text().catch(() => "")).slice(0, 300);- return { ok: false, reason: "upstream", message: `Gmail refused the send (${res.status}): ${text}` };+ try {+ await gmailApi(token, fetchImpl)("messages/send", {+ timeoutMs: 30_000,+ body: {+ raw: mime.toString("base64url"),+ ...(threadId ? { threadId } : {}),+ },+ });+ } catch (error) {+ if (!(error instanceof GmailApiError)) throw error;+ return { ok: false, reason: "upstream", message: `Gmail refused the send (${error.status}): ${error.detail}` };}return { ok: true, result: draft.body };}diff --git a/test/gmail-sent-route.test.ts b/test/gmail-sent-route.test.tsindex d2972319d..31d60b946 100644--- a/test/gmail-sent-route.test.ts+++ b/test/gmail-sent-route.test.ts@@ -87,3 +87,39 @@ test("sent mail resolves the connected Google account slot", async () => {accountType: "company",});});++test("Gmail upstream failures map to reconnect, not found, or retry responses without upstream bodies", async () => {+ const originalFetch = globalThis.fetch;+ try {+ for (const [upstream, status, message] of [+ [401, 409, "Reconnect Google to read sent mail."],+ [403, 409, "Reconnect Google to read sent mail."],+ [404, 404, "Couldn't load sent mail from Gmail. Try again."],+ [500, 502, "Couldn't load sent mail from Gmail. Try again."],+ ] as const) {+ let code = 0;+ let body = "";+ globalThis.fetch = async () => new Response("secret upstream detail", { status: upstream });+ await gmailSent({+ actor: { p: "[email protected]" },+ url: new URL("http://localhost/v1/connectors/gmail/sent/m1"),+ params: { messageId: "m1" },+ res: {+ setHeader() {},+ getHeader() {},+ writeHead(value: number) {+ code = value;+ },+ end(value: string) {+ body = value;+ },+ },+ deps: { connectorTokens: { connectorDerivedAuth: async () => ({ accessToken: "token" }) } },+ } as unknown as ApiCtx);+ assert.equal(code, status);+ assert.deepEqual(JSON.parse(body), { error: "gmail_unavailable", message });+ }+ } finally {+ globalThis.fetch = originalFetch;+ }+});diff --git a/test/gmail-sent.test.ts b/test/gmail-sent.test.tsindex 54aa278a7..440d1b91f 100644--- a/test/gmail-sent.test.ts+++ b/test/gmail-sent.test.ts@@ -1,6 +1,7 @@import assert from "node:assert/strict";import { test } from "node:test";-import { GmailReadError, getSentEmail, listSentEmails } from "../src/connectors/gmail-sent.ts";+import { getSentEmail, listSentEmails } from "../src/connectors/gmail-sent.ts";+import { GmailApiError } from "../src/connectors/gmail-api.ts";test("sent mail reads only SENT messages, paginates, and includes mail sent outside QM", async () => {const calls: URL[] = [];@@ -42,7 +43,7 @@ test("sent mail reads only SENT messages, paginates, and includes mail sent outstest("Gmail errors do not expose upstream response bodies or tokens", async () => {await assert.rejects(listSentEmails("private-token", undefined, async () => new Response("private upstream response", { status: 403 })),- (error: unknown) => error instanceof GmailReadError && error.status === 403 && !error.message.includes("private"),+ (error: unknown) => error instanceof GmailApiError && error.status === 403 && !error.message.includes("private"),);});@@ -63,6 +64,6 @@ test("sent message details decode MIME text and reject non-sent messages", asyncassert.deepEqual(result.attachments, ["report.pdf"]);await assert.rejects(getSentEmail("token", "m1", async () => Response.json({ ...payload, labelIds: ["INBOX"] })),- (error: unknown) => error instanceof GmailReadError && error.status === 404,+ (error: unknown) => error instanceof GmailApiError && error.status === 404,);});diff --git a/test/inbox-source-refresh.test.ts b/test/inbox-source-refresh.test.tsindex 6ef75ae61..020c6b916 100644--- a/test/inbox-source-refresh.test.ts+++ b/test/inbox-source-refresh.test.ts@@ -229,6 +229,25 @@ test("rate limiting is respected across different Slack items and never triggersassert.equal((await items.get(item.id))!.status, "ready");});+test("a Gmail rate limit pauses reads across the owner's Gmail items", async () => {+ const { items, item } = await fixture();+ let reads = 0;+ const refresh = createInboxSourceRefresh({+ featureFlags,+ items,+ tokens,+ directory,+ fetchImpl: async () => {+ reads++;+ return new Response("limited", { status: 429, headers: { "retry-after": "120" } });+ },+ });+ await refresh("owner", [item]);+ await refresh("owner", [{ ...item, id: "second" }]);+ assert.equal(reads, 1);+ assert.match(String((await items.get(item.id))!.sourcePayload?.sourceRefreshError), /not been dismissed/);+});+test("slow Gmail reads have a total batch budget and later cards are visited on the next refresh", async () => {const { items, item } = await fixture();let clock = 0;diff --git a/test/loop-source-send.test.ts b/test/loop-source-send.test.tsindex 7c98db6de..f75d751f3 100644--- a/test/loop-source-send.test.ts+++ b/test/loop-source-send.test.ts@@ -115,6 +115,24 @@ test("gmail send posts base64url raw with the thread id", async () => {assert.doesNotMatch(body.raw, /[+/=]/);});+test("a Gmail send refusal reports the upstream status and response excerpt", async () => {+ const result = await gmailAdapter.act(+ {+ owner: "josh",+ tokens: tokens({ "gmail.googleapis.com": "tok" }),+ fetchImpl: async () => new Response("Recipient address rejected", { status: 400 }),+ },+ gmailItem,+ "send",+ { body: "Confirmed." },+ );+ assert.deepEqual(result, {+ ok: false,+ reason: "upstream",+ message: "Gmail refused the send (400): Recipient address rejected",+ });+});+test("the held proposal is what goes out when the action carries no draft", async () => {let sent = "";const fetchImpl = (async (_url: Parameters<typeof fetch>[0], init?: RequestInit) => {