Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(examples): scope staging provisioning to selected entries
jkomyno9e8dc32d3dce
diff --git a/.github/workflows/examples-live.yml b/.github/workflows/examples-live.ymlindex f9e456cea..6abfe4554 100644--- a/.github/workflows/examples-live.yml+++ b/.github/workflows/examples-live.yml@@ -24,6 +24,10 @@ on:description: 'Comma-separated entry ids (empty = full manifest)'type: stringdefault: ''+ exclude_toolkits:+ description: 'Comma-separated toolkits to exclude, including dependent entries'+ type: string+ default: googledriveclient_tarball_url:description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)'type: string@@ -54,6 +58,9 @@ env:COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ # Keep unavailable coverage explicit. The harness records excluded entry ids+ # in its summary artifact; remove this default when staging has Drive auth.+ EXCLUDE_TOOLKITS: ${{ inputs.exclude_toolkits || 'googledrive' }}# mock unless explicitly dispatched live; the nightly adds a small live canary.LLM_MODE: ${{ inputs.llm || 'mock' }}# Entries mocks cannot cover (llmMock:false) plus one representative@@ -62,7 +69,7 @@ env:jobs:examples-live:- name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }})+ name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }}, excluding ${{ inputs.exclude_toolkits || 'googledrive' }})environment: stagingruns-on: ubuntu-latesttimeout-minutes: 60@@ -130,7 +137,19 @@ jobs:run: node harness/run.mjs selftest- name: Verify provisioned project state- run: node scripts/examples-provision.mjs --gc > /tmp/examples-provision.env+ env:+ IDS_INPUT: ${{ inputs.ids }}+ LANG_INPUT: ${{ inputs.lang || 'all' }}+ run: |+ set -euo pipefail+ ARGS=(--gc --exclude-toolkits "$EXCLUDE_TOOLKITS")+ if [ "$LANG_INPUT" != "all" ]; then+ ARGS+=(--lang "$LANG_INPUT")+ fi+ if [ -n "$IDS_INPUT" ]; then+ ARGS+=(--ids "$IDS_INPUT")+ fi+ node scripts/examples-provision.mjs "${ARGS[@]}" > /tmp/examples-provision.env- name: Sweepenv:@@ -148,6 +167,7 @@ jobs:if [ -n "$IDS_INPUT" ]; thenARGS+=(--ids "$IDS_INPUT")fi+ ARGS+=(--exclude-toolkits "$EXCLUDE_TOOLKITS")node harness/run.mjs sweep "${ARGS[@]}"- name: Live LLM canary@@ -155,7 +175,7 @@ jobs:run: |set -euo pipefailsource /tmp/examples-provision.env- node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS"+ node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS" --exclude-toolkits "$EXCLUDE_TOOLKITS"- name: Upload sweep results and tracesif: ${{ always() }}diff --git a/harness/manifest.mjs b/harness/manifest.mjsnew file mode 100644index 000000000..169743952--- /dev/null+++ b/harness/manifest.mjs@@ -0,0 +1,94 @@+import { readFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';++const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');++export const BROWSER_GRANT_TOOLKITS = [+ { exportPrefix: 'GMAIL', slug: 'gmail' },+ { slug: 'googledrive' },+ { exportPrefix: 'GITHUB', slug: 'github' },+ { exportPrefix: 'SLACK', slug: 'slack' },+];++export const DEMO_TOOLKIT = {+ exportPrefix: 'APIKEY',+ slug: 'serpapi',+ demoValue: 'examples-demo-key',+};++const parseCsv = value =>+ value+ ? value+ .split(',')+ .map(part => part.trim())+ .filter(Boolean)+ : [];++const toolkitForId = id => {+ for (const toolkit of [...BROWSER_GRANT_TOOLKITS, DEMO_TOOLKIT]) {+ if (toolkit.exportPrefix && id.startsWith(`COMPOSIO_EXAMPLES_${toolkit.exportPrefix}_`)) {+ return toolkit.slug;+ }+ }+ return undefined;+};++export const entryToolkits = entry => {+ const toolkits = new Set(entry.toolkits ?? []);+ for (const id of entry.ids ?? []) {+ const toolkit = toolkitForId(id);+ if (toolkit) toolkits.add(toolkit);+ }+ return [...toolkits];+};++export const loadManifest = () => {+ const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));+ for (const entry of manifest.entries) {+ if (!entry.id || !entry.lang || !entry.file || !entry.tier) {+ throw new Error(`manifest entry missing required fields: ${JSON.stringify(entry)}`);+ }+ if (entry.lang === 'ts' && entry.tier !== 'X' && !entry.pkg) {+ throw new Error(`ts entry ${entry.id} missing pkg`);+ }+ if (entry.tier === '3' && !entry.readiness) {+ throw new Error(`tier-3 entry ${entry.id} missing readiness regex`);+ }+ }+ return manifest.entries;+};++export const selectManifestEntries = (+ entries,+ { lang, ids, tiers = '1,2,3', excludeToolkits } = {}+) => {+ const selectedIds = new Set(parseCsv(ids));+ const selectedTiers = new Set(parseCsv(tiers));+ const excludedToolkitSet = new Set(parseCsv(excludeToolkits));+ const eligibleEntries = entries.filter(+ entry =>+ entry.tier !== 'X' &&+ selectedTiers.has(entry.tier) &&+ (!lang || entry.lang === lang) &&+ (selectedIds.size === 0 || selectedIds.has(entry.id))+ );+ const excludedEntries = eligibleEntries.filter(entry =>+ entryToolkits(entry).some(toolkit => excludedToolkitSet.has(toolkit))+ );+ const excludedIds = new Set(excludedEntries.map(entry => entry.id));++ return {+ entries: eligibleEntries.filter(entry => !excludedIds.has(entry.id)),+ excludedEntries,+ excludedToolkits: [...excludedToolkitSet],+ };+};++export const requiredBrowserGrantToolkits = entries =>+ BROWSER_GRANT_TOOLKITS.filter(toolkit =>+ entries.some(entry => entryToolkits(entry).includes(toolkit.slug))+ );++export const requiresDemoToolkit = entries =>+ entries.some(entry => entryToolkits(entry).includes(DEMO_TOOLKIT.slug));diff --git a/harness/run.mjs b/harness/run.mjsindex 50d583430..ff0c8eecc 100755--- a/harness/run.mjs+++ b/harness/run.mjs@@ -2,8 +2,8 @@// Examples runner: sweeps the entrypoints in examples-manifest.json against// the live backend, recording per-entry results and Composio traces.//-// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--llm live|mock]-// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--sample N] [--seed S]+// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--exclude-toolkits a,b] [--llm live|mock]+// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--exclude-toolkits a,b] [--sample N] [--seed S]// node harness/run.mjs selftest//// Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.@@ -12,6 +12,13 @@ import { spawn } from 'node:child_process';import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';import { dirname, join, relative, resolve } from 'node:path';import { fileURLToPath } from 'node:url';+import {+ entryToolkits,+ loadManifest,+ requiredBrowserGrantToolkits,+ requiresDemoToolkit,+ selectManifestEntries,+} from './manifest.mjs';import { resolveBackendBaseUrl, STAGING_BASE_URL } from './backend-url.mjs';const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');@@ -41,27 +48,21 @@ const fail = (msg, code = 2) => {throw new HarnessError(msg, code);};-const loadManifest = () => {- const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));- for (const e of manifest.entries) {- if (!e.id || !e.lang || !e.file || !e.tier) fail(`manifest entry missing required fields: ${JSON.stringify(e)}`);- if (e.lang === 'ts' && e.tier !== 'X' && !e.pkg) fail(`ts entry ${e.id} missing pkg`);- if (e.tier === '3' && !e.readiness) fail(`tier-3 entry ${e.id} missing readiness regex`);- }- return manifest.entries;-};+const selectionOptions = () => ({+ lang: opt('lang'),+ ids: opt('ids'),+ tiers: opt('tiers', '1,2,3'),+ excludeToolkits: opt('exclude-toolkits'),+});-const selectEntries = () => {- const lang = opt('lang');- const ids = opt('ids') ? opt('ids').split(',') : null;- const tiers = (opt('tiers', '1,2,3')).split(',');- return loadManifest().filter(- (e) =>- e.tier !== 'X' &&- tiers.includes(e.tier) &&- (!lang || e.lang === lang) &&- (!ids || ids.includes(e.id))+const selectEntries = () => selectManifestEntries(loadManifest(), selectionOptions());++const reportExclusions = selection => {+ if (selection.excludedEntries.length === 0) return;+ console.log(+ `excluding ${selection.excludedEntries.length} entries requiring ${selection.excludedToolkits.join(', ')}:`);+ for (const entry of selection.excludedEntries) console.log(` - ${entry.id}`);};const baseUrl = () => {@@ -389,9 +390,11 @@ const cmdSweep = async () => {const llm = opt('llm', 'live');if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);LLM_MOCK = llm === 'mock';- const entries = selectEntries();+ const selection = selectEntries();+ const entries = selection.entries;if (entries.length === 0) fail('no entries selected');if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);+ reportExclusions(selection);const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;const runDir = join(ARTIFACTS, id);@@ -431,7 +434,22 @@ const cmdSweep = async () => {};writeFileSync(join(runDir, 'summary.json'),- JSON.stringify({ runId: id, client, llm, baseUrl: baseUrl(), counts, startedAt: new Date().toISOString() }, null, 2)+ JSON.stringify(+ {+ runId: id,+ client,+ llm,+ baseUrl: baseUrl(),+ counts,+ selection: {+ excludedToolkits: selection.excludedToolkits,+ excludedEntries: selection.excludedEntries.map(entry => entry.id),+ },+ startedAt: new Date().toISOString(),+ },+ null,+ 2+ ));console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);process.exitCode = counts.red > 0 ? 1 : 0;@@ -443,7 +461,9 @@ const cmdSweep = async () => {};const cmdNeg = async () => {- let entries = selectEntries().filter((e) => e.backend !== false);+ const selection = selectEntries();+ reportExclusions(selection);+ let entries = selection.entries.filter((e) => e.backend !== false);const sample = opt('sample');if (sample) {const rand = mulberry32(Number(opt('seed', '42')));@@ -523,6 +543,41 @@ const cmdSelftest = async () => {}check('candidate swap rejects the installed baseline version', noOpCandidateRejected);+ const manifest = loadManifest();+ const withoutGoogleDrive = selectManifestEntries(manifest, {+ excludeToolkits: 'googledrive',+ });+ check(+ 'Google Drive exclusion removes every dependent entry and preserves the rest',+ withoutGoogleDrive.excludedEntries.length > 0 &&+ withoutGoogleDrive.excludedEntries.every(entry =>+ entryToolkits(entry).includes('googledrive')+ ) &&+ withoutGoogleDrive.entries.every(entry => !entryToolkits(entry).includes('googledrive')) &&+ withoutGoogleDrive.entries.length + withoutGoogleDrive.excludedEntries.length ===+ selectManifestEntries(manifest).entries.length+ );+ check(+ 'Google Drive exclusion removes it from provisioning',+ !requiredBrowserGrantToolkits(withoutGoogleDrive.entries).some(+ toolkit => toolkit.slug === 'googledrive'+ )+ );+ const apiKeyOnly = selectManifestEntries(manifest, {+ ids: 'ts/connected-accounts/api-key',+ }).entries;+ check(+ 'selected API-key example provisions only its demo toolkit',+ requiredBrowserGrantToolkits(apiKeyOnly).length === 0 && requiresDemoToolkit(apiKeyOnly)+ );+ const implicitGmail = selectManifestEntries(manifest, {+ ids: 'py/fastapi_app',+ }).entries;+ check(+ 'resource identifiers contribute implicit toolkit requirements',+ requiredBrowserGrantToolkits(implicitGmail).some(toolkit => toolkit.slug === 'gmail')+ );+const cleanupFixture = join(runDir, 'cleanup-fixture.txt');writeFileSync(cleanupFixture, 'user contents');const cleanupSnapshot = snapshotFiles([cleanupFixture]);diff --git a/scripts/examples-provision.mjs b/scripts/examples-provision.mjsindex 7950d7bee..ac662b860 100644--- a/scripts/examples-provision.mjs+++ b/scripts/examples-provision.mjs@@ -29,7 +29,20 @@// Auth config ids and connected account ids are not secrets; no credential// values are ever printed. The API-key demo value stored for serpapi is a// deliberately fake placeholder, not a real key.+//+// Selection flags match harness/run.mjs: [--lang ts|py] [--ids a,b]+// [--tiers 1,2,3] [--exclude-toolkits a,b]. Only resources required by the+// selected entries are provisioned; garbage collection remains project-wide+// within the examples-* ownership boundary above.+import {+ BROWSER_GRANT_TOOLKITS,+ DEMO_TOOLKIT,+ loadManifest,+ requiredBrowserGrantToolkits,+ requiresDemoToolkit,+ selectManifestEntries,+} from '../harness/manifest.mjs';import { resolveBackendBaseUrl } from '../harness/backend-url.mjs';let BASE_URL;@@ -44,6 +57,23 @@ const USER_ID = process.env.COMPOSIO_EXAMPLES_USER_ID ?? 'examples';const INITIATE_MISSING = process.argv.includes('--initiate-missing');const GC = process.argv.includes('--gc');const DRY_RUN = process.argv.includes('--dry-run');+const option = name => {+ const index = process.argv.indexOf(`--${name}`);+ return index >= 0 ? process.argv[index + 1] : undefined;+};++const selection = selectManifestEntries(loadManifest(), {+ lang: option('lang'),+ ids: option('ids'),+ tiers: option('tiers') ?? '1,2,3',+ excludeToolkits: option('exclude-toolkits'),+});+if (selection.entries.length === 0) {+ console.error('no example entries selected for provisioning');+ process.exit(1);+}+const selectedBrowserGrantToolkits = requiredBrowserGrantToolkits(selection.entries);+const needsDemoToolkit = requiresDemoToolkit(selection.entries);// COMPOSIO_BASE_URL selects the backend and defaults to staging;// resolveBackendBaseUrl refuses a structurally unusable one before the key is@@ -54,20 +84,15 @@ if (!API_KEY) {process.exit(1);}-// Toolkits the tier-2/3 entries depend on. OAuth toolkits need a one-time human-// browser authorization. Only export ids that examples consume directly.-const BROWSER_GRANT_TOOLKITS = [- { exportPrefix: 'GMAIL', slug: 'gmail' },- // googledrive exports nothing: examples reach Drive through the user's- // standing connection (COMPOSIO_EXAMPLES_USER_ID), never through ids.- { slug: 'googledrive' },- { exportPrefix: 'GITHUB', slug: 'github' },- { exportPrefix: 'SLACK', slug: 'slack' },-];-const DEMO_TOOLKIT = { exportPrefix: 'APIKEY', slug: 'serpapi', demoValue: 'examples-demo-key' };-const report = line => console.error(line);+if (selection.excludedEntries.length > 0) {+ report(+ `excluding ${selection.excludedEntries.length} entries requiring ${selection.excludedToolkits.join(', ')}:`+ );+ for (const entry of selection.excludedEntries) report(` - ${entry.id}`);+}+async function api(method, path, body) {const res = await fetch(`${BASE_URL}${path}`, {method,@@ -224,7 +249,7 @@ function findActiveAccount(slug, authConfigId) {);}-for (const { exportPrefix, slug } of BROWSER_GRANT_TOOLKITS) {+for (const { exportPrefix, slug } of selectedBrowserGrantToolkits) {let config = findAuthConfig(slug);if (!config) {const created = await api('POST', '/api/v3.1/auth_configs', {@@ -266,7 +291,7 @@ for (const { exportPrefix, slug } of BROWSER_GRANT_TOOLKITS) {// their own connected accounts to demonstrate that API. The stored value is a// placeholder; serpapi only validates it at tool-execution time and no example// executes a serpapi tool.-{+if (needsDemoToolkit) {const { exportPrefix, slug, demoValue } = DEMO_TOOLKIT;let config = findAuthConfig(slug);if (!config) {