Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(docs): retry 429s in data sync scripts and lower concurrency (#3608)
Alberto Schiabel82896858ecc3
diff --git a/docs/scripts/README.md b/docs/scripts/README.mdindex d6e947986..07dfbc2e5 100644--- a/docs/scripts/README.md+++ b/docs/scripts/README.md@@ -1,5 +1,19 @@# OpenAPI Scripts+## fetch-with-retry.ts++Shared `fetchWithRetry` helper used by the data-generation scripts+(`generate-toolkits.ts`, `generate-meta-tools.ts`). It wraps the global `fetch`+with rate-limit-aware retry/backoff: it retries `429` and transient 5xx+responses, honors the `Retry-After` header when present, otherwise falls back to+exponential backoff with jitter, and caps attempts so CI still fails fast when+the backend is genuinely down.++This matters because `generate-toolkits.ts` issues ~3000 requests per run+(2 + 3 per toolkit), which exceeds the staging limit of 2000 requests/minute.+Before this helper, runs failed with `429`, and `generate-meta-tools.ts` — which+runs immediately after — inherited the exhausted rate-limit window.+## fetch-openapi.mjsFetches the Composio OpenAPI spec and filters it for use in Fumadocs API reference documentation.diff --git a/docs/scripts/fetch-with-retry.ts b/docs/scripts/fetch-with-retry.tsnew file mode 100644index 000000000..24cad7cd2--- /dev/null+++ b/docs/scripts/fetch-with-retry.ts@@ -0,0 +1,139 @@+/**+ * Shared fetch helper for docs generation scripts.+ *+ * Wraps the global `fetch` with rate-limit-aware retry/backoff:+ * - Retries `429 Too Many Requests` and transient 5xx responses.+ * - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present.+ * - Falls back to exponential backoff with full jitter.+ * - Logs the endpoint and retry count so CI logs explain the delay.+ * - Caps attempts so CI still fails fast when the backend is genuinely down —+ * the final failing `Response` is returned to the caller, which keeps existing+ * `if (!response.ok)` handling intact.+ *+ * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried+ * the same way and re-thrown once attempts are exhausted.+ *+ * Why this exists: `generate-toolkits.ts` issues ~3000 requests per run, which+ * exceeds the staging limit of 2000 requests/minute. Without backoff the run+ * fails with `429`, and `generate-meta-tools.ts` (which runs next) inherits the+ * exhausted window. See docs/scripts/README.md.+ *+ * Deliberately hand-rolled rather than using Effect's `Schedule` (which would be+ * the idiomatic fit elsewhere): the docs package has no Effect dependency and+ * these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep+ * this helper dependency-free.+ */++const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];++export interface FetchWithRetryOptions extends RequestInit {+ /** Maximum number of attempts, including the first. Default 6. */+ maxAttempts?: number;+ /** Base delay in ms for exponential backoff. Default 1000. */+ baseDelayMs?: number;+ /** Upper bound for a single backoff wait in ms. Default 60000. */+ maxDelayMs?: number;+ /** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */+ retryStatuses?: number[];+}++function sleep(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++/** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */+function parseRetryAfter(header: string | null): number | null {+ if (!header) return null;++ const seconds = Number(header);+ if (Number.isFinite(seconds)) {+ return Math.max(0, seconds * 1000);+ }++ const dateMs = Date.parse(header);+ if (!Number.isNaN(dateMs)) {+ return Math.max(0, dateMs - Date.now());+ }++ return null;+}++/** Exponential backoff with full jitter. `attempt` is 1-based. */+function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {+ const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));+ return Math.random() * ceiling;+}++/** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */+function loggableEndpoint(url: string): string {+ try {+ const parsed = new URL(url);+ return `${parsed.origin}${parsed.pathname}`;+ } catch {+ return url;+ }+}++export async function fetchWithRetry(+ url: string,+ options: FetchWithRetryOptions = {}+): Promise<Response> {+ const {+ maxAttempts = 6,+ baseDelayMs = 1000,+ maxDelayMs = 60_000,+ retryStatuses = DEFAULT_RETRY_STATUSES,+ ...init+ } = options;++ const endpoint = loggableEndpoint(url);+ let lastError: unknown;++ for (let attempt = 1; attempt <= maxAttempts; attempt++) {+ let response: Response;++ try {+ response = await fetch(url, init);+ } catch (error) {+ lastError = error;+ if (attempt >= maxAttempts) break;+ const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs);+ console.warn(+ ` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` ++ `retrying in ${Math.round(delay)}ms: ${(error as Error).message}`+ );+ await sleep(delay);+ continue;+ }++ // Success, or a non-retryable status: hand it back to the caller.+ if (!retryStatuses.includes(response.status)) {+ return response;+ }++ // Out of attempts: surface the failing response so callers still fail.+ if (attempt >= maxAttempts) {+ return response;+ }++ const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));+ // Add jitter on top of Retry-After to avoid a synchronized retry burst.+ const delay =+ retryAfterMs != null+ ? retryAfterMs + Math.random() * 1000+ : backoffDelay(attempt, baseDelayMs, maxDelayMs);++ console.warn(+ ` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` ++ `retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}`+ );++ // Drain the body so the connection can be reused.+ await response.arrayBuffer().catch(() => {});+ await sleep(delay);+ }++ throw lastError instanceof Error+ ? lastError+ : new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`);+}diff --git a/docs/scripts/generate-meta-tools.ts b/docs/scripts/generate-meta-tools.tsindex 93f58a6bc..7f1861b4b 100644--- a/docs/scripts/generate-meta-tools.ts+++ b/docs/scripts/generate-meta-tools.ts@@ -18,6 +18,7 @@import { writeFile, mkdir, readdir, unlink } from 'fs/promises';import { join } from 'path';+import { fetchWithRetry } from './fetch-with-retry';const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';const API_KEY = process.env.COMPOSIO_API_KEY;@@ -33,7 +34,7 @@ const CONTENT_DIR = join(process.cwd(), 'content/reference/meta-tools');async function createSession(): Promise<string> {console.log('Creating session...');- const response = await fetch(`${API_BASE}/tool_router/session`, {+ const response = await fetchWithRetry(`${API_BASE}/tool_router/session`, {method: 'POST',headers: {'Content-Type': 'application/json',@@ -69,7 +70,7 @@ async function createSession(): Promise<string> {async function fetchMetaTools(sessionId: string): Promise<any[]> {console.log('Fetching meta tools with schemas...');- const response = await fetch(`${API_BASE}/tool_router/session/${sessionId}/tools`, {+ const response = await fetchWithRetry(`${API_BASE}/tool_router/session/${sessionId}/tools`, {headers: {'x-api-key': API_KEY!,},diff --git a/docs/scripts/generate-toolkits.ts b/docs/scripts/generate-toolkits.tsindex 1a61e5877..b1bf1ae77 100644--- a/docs/scripts/generate-toolkits.ts+++ b/docs/scripts/generate-toolkits.ts@@ -10,6 +10,7 @@import { mkdir, writeFile } from 'fs/promises';import { join } from 'path';+import { fetchWithRetry } from './fetch-with-retry';const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';const API_KEY = process.env.COMPOSIO_API_KEY;@@ -76,7 +77,7 @@ interface Toolkit {async function fetchToolkits(): Promise<any[]> {console.log('Fetching toolkits from API...');- const response = await fetch(`${API_BASE}/toolkits`, {+ const response = await fetchWithRetry(`${API_BASE}/toolkits`, {headers: {'Content-Type': 'application/json','x-api-key': API_KEY!,@@ -94,7 +95,7 @@ async function fetchToolkits(): Promise<any[]> {async function fetchToolkitChangelog(): Promise<Map<string, string>> {console.log('Fetching toolkit changelog...');- const response = await fetch(`${API_BASE}/toolkits/changelog`, {+ const response = await fetchWithRetry(`${API_BASE}/toolkits/changelog`, {headers: {'Content-Type': 'application/json','x-api-key': API_KEY!,@@ -124,7 +125,7 @@ async function fetchToolkitChangelog(): Promise<Map<string, string>> {}async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {- const response = await fetch(`${API_BASE}/tools?toolkit_slug=${slug}&toolkit_versions=latest&limit=1000`, {+ const response = await fetchWithRetry(`${API_BASE}/tools?toolkit_slug=${slug}&toolkit_versions=latest&limit=1000`, {headers: {'Content-Type': 'application/json','x-api-key': API_KEY!,@@ -145,7 +146,7 @@ async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {}async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {- const response = await fetch(`${API_BASE}/triggers_types?toolkit_slugs=${slug}&toolkit_versions=latest`, {+ const response = await fetchWithRetry(`${API_BASE}/triggers_types?toolkit_slugs=${slug}&toolkit_versions=latest`, {headers: {'Content-Type': 'application/json','x-api-key': API_KEY!,@@ -166,7 +167,7 @@ async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {}async function fetchAuthConfigDetails(slug: string): Promise<AuthConfigDetail[]> {- const response = await fetch(`${API_BASE}/toolkits/${slug}`, {+ const response = await fetchWithRetry(`${API_BASE}/toolkits/${slug}`, {headers: {'Content-Type': 'application/json','x-api-key': API_KEY!,@@ -263,9 +264,12 @@ async function main() {toolkit.version = versionMap.get(toolkit.slug) || null;}- // Fetch tools, triggers, and auth config details for each toolkit in batches+ // Fetch tools, triggers, and auth config details for each toolkit in batches.+ // Each toolkit fires 3 requests in parallel, so batchSize N = ~3N concurrent+ // requests. Kept low to soften burst pressure on the staging rate limit+ // (2000 req/min); fetchWithRetry handles the remaining 429s with backoff.console.log('Fetching tools, triggers, and auth config details...');- const batchSize = 10;+ const batchSize = 5;let completed = 0;for (let i = 0; i < toolkits.length; i += batchSize) {