Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(examples): scope staging provisioning to selected entries

jkomyno9e8dc32d3dce
diff --git a/.github/workflows/examples-live.yml b/.github/workflows/examples-live.yml
index f9e456cea..6abfe4554 100644
--- a/.github/workflows/examples-live.yml
+++ b/.github/workflows/examples-live.yml
@@ -24,6 +24,10 @@ on:
description: 'Comma-separated entry ids (empty = full manifest)'
type: string
default: ''
+ exclude_toolkits:
+ description: 'Comma-separated toolkits to exclude, including dependent entries'
+ type: string
+ default: googledrive
client_tarball_url:
description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)'
type: string
@@ -54,6 +58,9 @@ env:
COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ # Keep unavailable coverage explicit. The harness records excluded entry ids
+ # in its summary artifact; remove this default when staging has Drive auth.
+ EXCLUDE_TOOLKITS: ${{ inputs.exclude_toolkits || 'googledrive' }}
# mock unless explicitly dispatched live; the nightly adds a small live canary.
LLM_MODE: ${{ inputs.llm || 'mock' }}
# Entries mocks cannot cover (llmMock:false) plus one representative
@@ -62,7 +69,7 @@ env:
jobs:
examples-live:
- name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }})
+ name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }}, excluding ${{ inputs.exclude_toolkits || 'googledrive' }})
environment: staging
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -130,7 +137,19 @@ jobs:
run: node harness/run.mjs selftest
- name: Verify provisioned project state
- run: node scripts/examples-provision.mjs --gc > /tmp/examples-provision.env
+ env:
+ IDS_INPUT: ${{ inputs.ids }}
+ LANG_INPUT: ${{ inputs.lang || 'all' }}
+ run: |
+ set -euo pipefail
+ ARGS=(--gc --exclude-toolkits "$EXCLUDE_TOOLKITS")
+ if [ "$LANG_INPUT" != "all" ]; then
+ ARGS+=(--lang "$LANG_INPUT")
+ fi
+ if [ -n "$IDS_INPUT" ]; then
+ ARGS+=(--ids "$IDS_INPUT")
+ fi
+ node scripts/examples-provision.mjs "${ARGS[@]}" > /tmp/examples-provision.env
- name: Sweep
env:
@@ -148,6 +167,7 @@ jobs:
if [ -n "$IDS_INPUT" ]; then
ARGS+=(--ids "$IDS_INPUT")
fi
+ ARGS+=(--exclude-toolkits "$EXCLUDE_TOOLKITS")
node harness/run.mjs sweep "${ARGS[@]}"
- name: Live LLM canary
@@ -155,7 +175,7 @@ jobs:
run: |
set -euo pipefail
source /tmp/examples-provision.env
- node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS"
+ node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS" --exclude-toolkits "$EXCLUDE_TOOLKITS"
- name: Upload sweep results and traces
if: ${{ always() }}
diff --git a/harness/manifest.mjs b/harness/manifest.mjs
new file mode 100644
index 000000000..169743952
--- /dev/null
+++ b/harness/manifest.mjs
@@ -0,0 +1,94 @@
+import { readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+
+export const BROWSER_GRANT_TOOLKITS = [
+ { exportPrefix: 'GMAIL', slug: 'gmail' },
+ { slug: 'googledrive' },
+ { exportPrefix: 'GITHUB', slug: 'github' },
+ { exportPrefix: 'SLACK', slug: 'slack' },
+];
+
+export const DEMO_TOOLKIT = {
+ exportPrefix: 'APIKEY',
+ slug: 'serpapi',
+ demoValue: 'examples-demo-key',
+};
+
+const parseCsv = value =>
+ value
+ ? value
+ .split(',')
+ .map(part => part.trim())
+ .filter(Boolean)
+ : [];
+
+const toolkitForId = id => {
+ for (const toolkit of [...BROWSER_GRANT_TOOLKITS, DEMO_TOOLKIT]) {
+ if (toolkit.exportPrefix && id.startsWith(`COMPOSIO_EXAMPLES_${toolkit.exportPrefix}_`)) {
+ return toolkit.slug;
+ }
+ }
+ return undefined;
+};
+
+export const entryToolkits = entry => {
+ const toolkits = new Set(entry.toolkits ?? []);
+ for (const id of entry.ids ?? []) {
+ const toolkit = toolkitForId(id);
+ if (toolkit) toolkits.add(toolkit);
+ }
+ return [...toolkits];
+};
+
+export const loadManifest = () => {
+ const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));
+ for (const entry of manifest.entries) {
+ if (!entry.id || !entry.lang || !entry.file || !entry.tier) {
+ throw new Error(`manifest entry missing required fields: ${JSON.stringify(entry)}`);
+ }
+ if (entry.lang === 'ts' && entry.tier !== 'X' && !entry.pkg) {
+ throw new Error(`ts entry ${entry.id} missing pkg`);
+ }
+ if (entry.tier === '3' && !entry.readiness) {
+ throw new Error(`tier-3 entry ${entry.id} missing readiness regex`);
+ }
+ }
+ return manifest.entries;
+};
+
+export const selectManifestEntries = (
+ entries,
+ { lang, ids, tiers = '1,2,3', excludeToolkits } = {}
+) => {
+ const selectedIds = new Set(parseCsv(ids));
+ const selectedTiers = new Set(parseCsv(tiers));
+ const excludedToolkitSet = new Set(parseCsv(excludeToolkits));
+ const eligibleEntries = entries.filter(
+ entry =>
+ entry.tier !== 'X' &&
+ selectedTiers.has(entry.tier) &&
+ (!lang || entry.lang === lang) &&
+ (selectedIds.size === 0 || selectedIds.has(entry.id))
+ );
+ const excludedEntries = eligibleEntries.filter(entry =>
+ entryToolkits(entry).some(toolkit => excludedToolkitSet.has(toolkit))
+ );
+ const excludedIds = new Set(excludedEntries.map(entry => entry.id));
+
+ return {
+ entries: eligibleEntries.filter(entry => !excludedIds.has(entry.id)),
+ excludedEntries,
+ excludedToolkits: [...excludedToolkitSet],
+ };
+};
+
+export const requiredBrowserGrantToolkits = entries =>
+ BROWSER_GRANT_TOOLKITS.filter(toolkit =>
+ entries.some(entry => entryToolkits(entry).includes(toolkit.slug))
+ );
+
+export const requiresDemoToolkit = entries =>
+ entries.some(entry => entryToolkits(entry).includes(DEMO_TOOLKIT.slug));
diff --git a/harness/run.mjs b/harness/run.mjs
index 50d583430..ff0c8eecc 100755
--- a/harness/run.mjs
+++ b/harness/run.mjs
@@ -2,8 +2,8 @@
// Examples runner: sweeps the entrypoints in examples-manifest.json against
// the live backend, recording per-entry results and Composio traces.
//
-// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--llm live|mock]
-// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--sample N] [--seed S]
+// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--exclude-toolkits a,b] [--llm live|mock]
+// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--exclude-toolkits a,b] [--sample N] [--seed S]
// node harness/run.mjs selftest
//
// Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.
@@ -12,6 +12,13 @@ import { spawn } from 'node:child_process';
import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';
import { dirname, join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
+import {
+ entryToolkits,
+ loadManifest,
+ requiredBrowserGrantToolkits,
+ requiresDemoToolkit,
+ selectManifestEntries,
+} from './manifest.mjs';
import { resolveBackendBaseUrl, STAGING_BASE_URL } from './backend-url.mjs';
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
@@ -41,27 +48,21 @@ const fail = (msg, code = 2) => {
throw new HarnessError(msg, code);
};
-const loadManifest = () => {
- const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));
- for (const e of manifest.entries) {
- if (!e.id || !e.lang || !e.file || !e.tier) fail(`manifest entry missing required fields: ${JSON.stringify(e)}`);
- if (e.lang === 'ts' && e.tier !== 'X' && !e.pkg) fail(`ts entry ${e.id} missing pkg`);
- if (e.tier === '3' && !e.readiness) fail(`tier-3 entry ${e.id} missing readiness regex`);
- }
- return manifest.entries;
-};
+const selectionOptions = () => ({
+ lang: opt('lang'),
+ ids: opt('ids'),
+ tiers: opt('tiers', '1,2,3'),
+ excludeToolkits: opt('exclude-toolkits'),
+});
-const selectEntries = () => {
- const lang = opt('lang');
- const ids = opt('ids') ? opt('ids').split(',') : null;
- const tiers = (opt('tiers', '1,2,3')).split(',');
- return loadManifest().filter(
- (e) =>
- e.tier !== 'X' &&
- tiers.includes(e.tier) &&
- (!lang || e.lang === lang) &&
- (!ids || ids.includes(e.id))
+const selectEntries = () => selectManifestEntries(loadManifest(), selectionOptions());
+
+const reportExclusions = selection => {
+ if (selection.excludedEntries.length === 0) return;
+ console.log(
+ `excluding ${selection.excludedEntries.length} entries requiring ${selection.excludedToolkits.join(', ')}:`
);
+ for (const entry of selection.excludedEntries) console.log(` - ${entry.id}`);
};
const baseUrl = () => {
@@ -389,9 +390,11 @@ const cmdSweep = async () => {
const llm = opt('llm', 'live');
if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);
LLM_MOCK = llm === 'mock';
- const entries = selectEntries();
+ const selection = selectEntries();
+ const entries = selection.entries;
if (entries.length === 0) fail('no entries selected');
if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);
+ reportExclusions(selection);
const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;
const runDir = join(ARTIFACTS, id);
@@ -431,7 +434,22 @@ const cmdSweep = async () => {
};
writeFileSync(
join(runDir, 'summary.json'),
- JSON.stringify({ runId: id, client, llm, baseUrl: baseUrl(), counts, startedAt: new Date().toISOString() }, null, 2)
+ JSON.stringify(
+ {
+ runId: id,
+ client,
+ llm,
+ baseUrl: baseUrl(),
+ counts,
+ selection: {
+ excludedToolkits: selection.excludedToolkits,
+ excludedEntries: selection.excludedEntries.map(entry => entry.id),
+ },
+ startedAt: new Date().toISOString(),
+ },
+ null,
+ 2
+ )
);
console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);
process.exitCode = counts.red > 0 ? 1 : 0;
@@ -443,7 +461,9 @@ const cmdSweep = async () => {
};
const cmdNeg = async () => {
- let entries = selectEntries().filter((e) => e.backend !== false);
+ const selection = selectEntries();
+ reportExclusions(selection);
+ let entries = selection.entries.filter((e) => e.backend !== false);
const sample = opt('sample');
if (sample) {
const rand = mulberry32(Number(opt('seed', '42')));
@@ -523,6 +543,41 @@ const cmdSelftest = async () => {
}
check('candidate swap rejects the installed baseline version', noOpCandidateRejected);
+ const manifest = loadManifest();
+ const withoutGoogleDrive = selectManifestEntries(manifest, {
+ excludeToolkits: 'googledrive',
+ });
+ check(
+ 'Google Drive exclusion removes every dependent entry and preserves the rest',
+ withoutGoogleDrive.excludedEntries.length > 0 &&
+ withoutGoogleDrive.excludedEntries.every(entry =>
+ entryToolkits(entry).includes('googledrive')
+ ) &&
+ withoutGoogleDrive.entries.every(entry => !entryToolkits(entry).includes('googledrive')) &&
+ withoutGoogleDrive.entries.length + withoutGoogleDrive.excludedEntries.length ===
+ selectManifestEntries(manifest).entries.length
+ );
+ check(
+ 'Google Drive exclusion removes it from provisioning',
+ !requiredBrowserGrantToolkits(withoutGoogleDrive.entries).some(
+ toolkit => toolkit.slug === 'googledrive'
+ )
+ );
+ const apiKeyOnly = selectManifestEntries(manifest, {
+ ids: 'ts/connected-accounts/api-key',
+ }).entries;
+ check(
+ 'selected API-key example provisions only its demo toolkit',
+ requiredBrowserGrantToolkits(apiKeyOnly).length === 0 && requiresDemoToolkit(apiKeyOnly)
+ );
+ const implicitGmail = selectManifestEntries(manifest, {
+ ids: 'py/fastapi_app',
+ }).entries;
+ check(
+ 'resource identifiers contribute implicit toolkit requirements',
+ requiredBrowserGrantToolkits(implicitGmail).some(toolkit => toolkit.slug === 'gmail')
+ );
+
const cleanupFixture = join(runDir, 'cleanup-fixture.txt');
writeFileSync(cleanupFixture, 'user contents');
const cleanupSnapshot = snapshotFiles([cleanupFixture]);
diff --git a/scripts/examples-provision.mjs b/scripts/examples-provision.mjs
index 7950d7bee..ac662b860 100644
--- a/scripts/examples-provision.mjs
+++ b/scripts/examples-provision.mjs
@@ -29,7 +29,20 @@
// Auth config ids and connected account ids are not secrets; no credential
// values are ever printed. The API-key demo value stored for serpapi is a
// deliberately fake placeholder, not a real key.
+//
+// Selection flags match harness/run.mjs: [--lang ts|py] [--ids a,b]
+// [--tiers 1,2,3] [--exclude-toolkits a,b]. Only resources required by the
+// selected entries are provisioned; garbage collection remains project-wide
+// within the examples-* ownership boundary above.
+import {
+ BROWSER_GRANT_TOOLKITS,
+ DEMO_TOOLKIT,
+ loadManifest,
+ requiredBrowserGrantToolkits,
+ requiresDemoToolkit,
+ selectManifestEntries,
+} from '../harness/manifest.mjs';
import { resolveBackendBaseUrl } from '../harness/backend-url.mjs';
let BASE_URL;
@@ -44,6 +57,23 @@ const USER_ID = process.env.COMPOSIO_EXAMPLES_USER_ID ?? 'examples';
const INITIATE_MISSING = process.argv.includes('--initiate-missing');
const GC = process.argv.includes('--gc');
const DRY_RUN = process.argv.includes('--dry-run');
+const option = name => {
+ const index = process.argv.indexOf(`--${name}`);
+ return index >= 0 ? process.argv[index + 1] : undefined;
+};
+
+const selection = selectManifestEntries(loadManifest(), {
+ lang: option('lang'),
+ ids: option('ids'),
+ tiers: option('tiers') ?? '1,2,3',
+ excludeToolkits: option('exclude-toolkits'),
+});
+if (selection.entries.length === 0) {
+ console.error('no example entries selected for provisioning');
+ process.exit(1);
+}
+const selectedBrowserGrantToolkits = requiredBrowserGrantToolkits(selection.entries);
+const needsDemoToolkit = requiresDemoToolkit(selection.entries);
// COMPOSIO_BASE_URL selects the backend and defaults to staging;
// resolveBackendBaseUrl refuses a structurally unusable one before the key is
@@ -54,20 +84,15 @@ if (!API_KEY) {
process.exit(1);
}
-// Toolkits the tier-2/3 entries depend on. OAuth toolkits need a one-time human
-// browser authorization. Only export ids that examples consume directly.
-const BROWSER_GRANT_TOOLKITS = [
- { exportPrefix: 'GMAIL', slug: 'gmail' },
- // googledrive exports nothing: examples reach Drive through the user's
- // standing connection (COMPOSIO_EXAMPLES_USER_ID), never through ids.
- { slug: 'googledrive' },
- { exportPrefix: 'GITHUB', slug: 'github' },
- { exportPrefix: 'SLACK', slug: 'slack' },
-];
-const DEMO_TOOLKIT = { exportPrefix: 'APIKEY', slug: 'serpapi', demoValue: 'examples-demo-key' };
-
const report = line => console.error(line);
+if (selection.excludedEntries.length > 0) {
+ report(
+ `excluding ${selection.excludedEntries.length} entries requiring ${selection.excludedToolkits.join(', ')}:`
+ );
+ for (const entry of selection.excludedEntries) report(` - ${entry.id}`);
+}
+
async function api(method, path, body) {
const res = await fetch(`${BASE_URL}${path}`, {
method,
@@ -224,7 +249,7 @@ function findActiveAccount(slug, authConfigId) {
);
}
-for (const { exportPrefix, slug } of BROWSER_GRANT_TOOLKITS) {
+for (const { exportPrefix, slug } of selectedBrowserGrantToolkits) {
let config = findAuthConfig(slug);
if (!config) {
const created = await api('POST', '/api/v3.1/auth_configs', {
@@ -266,7 +291,7 @@ for (const { exportPrefix, slug } of BROWSER_GRANT_TOOLKITS) {
// their own connected accounts to demonstrate that API. The stored value is a
// placeholder; serpapi only validates it at tool-execution time and no example
// executes a serpapi tool.
-{
+if (needsDemoToolkit) {
const { exportPrefix, slug, demoValue } = DEMO_TOOLKIT;
let config = findAuthConfig(slug);
if (!config) {