Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

feat(examples): live-sweep harness, manifest, and scheduled workflow (#4117)

Alberto Schiabel71ec523d258b
diff --git a/.github/workflows/examples-live.yml b/.github/workflows/examples-live.yml
new file mode 100644
index 000000000..f9e456cea
--- /dev/null
+++ b/.github/workflows/examples-live.yml
@@ -0,0 +1,167 @@
+name: Examples Manifest Sweep (staging)
+
+# Runs the example entrypoints from examples-manifest.json against the
+# STAGING backend using a dedicated, disposable examples project — never
+# the org's main key. Non-blocking by design: scheduled + on demand, never a
+# required PR check. Local runs and CI share the same code path
+# (harness/run.mjs sweep).
+on:
+ schedule:
+ - cron: '0 7 * * *' # 07:00 UTC nightly
+ workflow_dispatch:
+ inputs:
+ lang:
+ description: 'Language to sweep'
+ type: choice
+ options: [all, ts, py]
+ default: all
+ client:
+ description: 'Client under test'
+ type: choice
+ options: [baseline, candidate]
+ default: baseline
+ ids:
+ description: 'Comma-separated entry ids (empty = full manifest)'
+ type: string
+ default: ''
+ client_tarball_url:
+ description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)'
+ type: string
+ default: ''
+ client_wheel_url:
+ description: 'Candidate Python client wheel URL (required for Python candidate entries)'
+ type: string
+ default: ''
+ llm:
+ description: 'LLM traffic: mock (aimock, zero model spend) or live'
+ type: choice
+ options: [mock, live]
+ default: mock
+
+concurrency:
+ group: ${{ github.workflow }}
+ cancel-in-progress: false
+
+permissions:
+ contents: read
+
+env:
+ # Staging URL is not a secret; the keys are. The key must belong to a
+ # project that holds no valuable connections or data — never the org's
+ # main project. Example runs create their own resources and only ever
+ # delete what they created.
+ COMPOSIO_BASE_URL: https://staging-backend.composio.dev
+ COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ # mock unless explicitly dispatched live; the nightly adds a small live canary.
+ LLM_MODE: ${{ inputs.llm || 'mock' }}
+ # Entries mocks cannot cover (llmMock:false) plus one representative
+ # tool-calling example per language: the drift signal mocks can't see.
+ CANARY_IDS: ts/openai/agents-api,ts/openai/agents-api-tool-router,ts/openai/chat-completions,py/tool_router/langchain_agent
+
+jobs:
+ examples-live:
+ name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }})
+ environment: staging
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+
+ steps:
+ - name: Checkout Code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+ - name: Setup Node.js, pnpm, Bun
+ uses: ./.github/actions/setup-node-pnpm-bun
+ with:
+ enable-turbo-cache: 'true'
+
+ - name: Setup Python and uv
+ uses: ./.github/actions/setup-python-uv
+
+ - name: Require credentials
+ run: |
+ if [ -z "$COMPOSIO_API_KEY" ]; then
+ echo "::error::COMPOSIO_API_KEY is required."
+ exit 1
+ fi
+ # LLM keys are only needed when live model traffic will run:
+ # an explicit live dispatch, or the nightly's live canary step.
+ if [ "$LLM_MODE" = "live" ] || [ "${{ github.event_name }}" = "schedule" ]; then
+ if [ -z "$OPENAI_API_KEY" ] || [ -z "$ANTHROPIC_API_KEY" ]; then
+ echo "::error::OPENAI_API_KEY and ANTHROPIC_API_KEY are required for live LLM runs."
+ exit 1
+ fi
+ fi
+
+ - name: Install Dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Build SDK packages
+ run: pnpm run build:packages
+
+ - name: Fetch candidate client artifacts
+ if: ${{ inputs.client == 'candidate' }}
+ env:
+ CLIENT_TARBALL_URL: ${{ inputs.client_tarball_url }}
+ CLIENT_WHEEL_URL: ${{ inputs.client_wheel_url }}
+ LANG_INPUT: ${{ inputs.lang || 'all' }}
+ run: |
+ set -euo pipefail
+ mkdir -p .artifacts/candidate
+ if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "ts" ]; then
+ if [[ "$CLIENT_TARBALL_URL" != https://* ]]; then
+ echo "::error::TS candidate entries require an HTTPS client_tarball_url"
+ exit 1
+ fi
+ curl -fsSL -o .artifacts/candidate/composio-client.tgz "$CLIENT_TARBALL_URL"
+ echo "COMPOSIO_CLIENT_TARBALL=$PWD/.artifacts/candidate/composio-client.tgz" >> "$GITHUB_ENV"
+ fi
+ if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "py" ]; then
+ if [[ "$CLIENT_WHEEL_URL" != https://* ]]; then
+ echo "::error::Python candidate entries require an HTTPS client_wheel_url"
+ exit 1
+ fi
+ curl -fsSL -o .artifacts/candidate/composio-client.whl "$CLIENT_WHEEL_URL"
+ echo "COMPOSIO_CLIENT_WHEEL=$PWD/.artifacts/candidate/composio-client.whl" >> "$GITHUB_ENV"
+ fi
+
+ - name: Harness selftest
+ run: node harness/run.mjs selftest
+
+ - name: Verify provisioned project state
+ run: node scripts/examples-provision.mjs --gc > /tmp/examples-provision.env
+
+ - name: Sweep
+ env:
+ CLIENT_INPUT: ${{ inputs.client || 'baseline' }}
+ IDS_INPUT: ${{ inputs.ids }}
+ LANG_INPUT: ${{ inputs.lang || 'all' }}
+ run: |
+ set -euo pipefail
+ # Load the provisioned COMPOSIO_EXAMPLES_* ids emitted above.
+ source /tmp/examples-provision.env
+ ARGS=(--client "$CLIENT_INPUT" --llm "$LLM_MODE")
+ if [ "$LANG_INPUT" != "all" ]; then
+ ARGS+=(--lang "$LANG_INPUT")
+ fi
+ if [ -n "$IDS_INPUT" ]; then
+ ARGS+=(--ids "$IDS_INPUT")
+ fi
+ node harness/run.mjs sweep "${ARGS[@]}"
+
+ - name: Live LLM canary
+ if: ${{ github.event_name == 'schedule' }}
+ run: |
+ set -euo pipefail
+ source /tmp/examples-provision.env
+ node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS"
+
+ - name: Upload sweep results and traces
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: examples-live-results
+ path: .artifacts/examples-parity/
+ retention-days: 14
+ if-no-files-found: ignore
diff --git a/examples-manifest.json b/examples-manifest.json
new file mode 100644
index 000000000..aae276a32
--- /dev/null
+++ b/examples-manifest.json
@@ -0,0 +1,1294 @@
+{
+ "$comment": "Inventory of runnable example entrypoints, consumed by harness/run.mjs. Tiers: 1=unattended, 2=needs provisioned state (scripts/examples-provision.mjs), 3=bounded (pass at readiness match, then terminate), X=excluded. Readiness regexes must anchor on a labeled output line the example prints on success (e.g. 'Visit this URL to authorize: https?://'), never a bare 'https?://' — a URL inside an error message must not count as ready. llmMock:false marks entries whose LLM traffic cannot run against the mock server (live canary only).",
+ "entries": [
+ {
+ "id": "ts/anthropic/index",
+ "lang": "ts",
+ "pkg": "anthropic-example",
+ "file": "ts/examples/anthropic/src/index.ts",
+ "tier": "1",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/anthropic/streaming",
+ "lang": "ts",
+ "pkg": "anthropic-example",
+ "file": "ts/examples/anthropic/src/streaming.ts",
+ "tier": "1",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/anthropic/mcp",
+ "lang": "ts",
+ "pkg": "anthropic-example",
+ "file": "ts/examples/anthropic/src/mcp.ts",
+ "tier": "2",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/anthropic/experimental.mcp",
+ "lang": "ts",
+ "pkg": "anthropic-example",
+ "file": "ts/examples/anthropic/src/experimental.mcp.ts",
+ "tier": "2",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/anthropic/claude-agents-sdk",
+ "lang": "ts",
+ "pkg": "anthropic-example",
+ "file": "ts/examples/anthropic/src/claude-agents-sdk.ts",
+ "tier": "2",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 300,
+ "notes": "needs @anthropic-ai/claude-code CLI on PATH"
+ },
+ {
+ "id": "ts/cloudflare-wrangler/dev",
+ "lang": "ts",
+ "pkg": "cloudflare-wrangler-example",
+ "file": "ts/examples/cloudflare-wrangler/src/index.ts",
+ "tier": "3",
+ "script": "dev",
+ "readiness": "Ready on|Listening",
+ "toolkits": [
+ "googledrive"
+ ],
+ "timeoutSec": 120,
+ "notes": "wrangler dev; env must reach .dev.vars"
+ },
+ {
+ "id": "ts/connected-accounts/index",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/index.ts",
+ "tier": "3",
+ "readiness": "authorize the user: https?://",
+ "toolkits": [
+ "github"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/connected-accounts/api-key",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/api-key.ts",
+ "tier": "2",
+ "ids": [
+ "COMPOSIO_EXAMPLES_APIKEY_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_APIKEY_PLACEHOLDER",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/connected-accounts/magic-flow-demo",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/magic-flow-demo.ts",
+ "tier": "3",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "readiness": "Redirect the user to https?://",
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/connected-accounts/multiple-connected-accounts",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/multiple-connected-accounts.ts",
+ "tier": "2",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/connected-accounts/scopes",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/scopes.ts",
+ "tier": "2",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/connected-accounts/toolkit-authorize",
+ "lang": "ts",
+ "pkg": "connected-accounts-example",
+ "file": "ts/examples/connected-accounts/src/toolkit-authorize.ts",
+ "tier": "3",
+ "readiness": "authorize the toolkit: https?://",
+ "toolkits": [
+ "github"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/error-handling-demo/index",
+ "lang": "ts",
+ "pkg": "error-handling-example",
+ "file": "ts/examples/error-handling-demo/src/index.ts",
+ "tier": "1",
+ "backend": false,
+ "timeoutSec": 60
+ },
+ {
+ "id": "ts/file-handling/index",
+ "lang": "ts",
+ "pkg": "file-handling-example",
+ "file": "ts/examples/file-handling/src/index.ts",
+ "tier": "X",
+ "toolkits": [
+ "gmail"
+ ],
+ "notes": "sends real email by design; excluded until a non-send variant is decided by the human",
+ "reason": "outbound-email"
+ },
+ {
+ "id": "ts/file-handling/url",
+ "lang": "ts",
+ "pkg": "file-handling-example",
+ "file": "ts/examples/file-handling/src/url.ts",
+ "tier": "X",
+ "toolkits": [
+ "gmail"
+ ],
+ "reason": "outbound-email",
+ "notes": "sends real email by design; excluded until a non-send variant is decided by the human"
+ },
+ {
+ "id": "ts/google/index",
+ "lang": "ts",
+ "pkg": "google-example",
+ "file": "ts/examples/google/src/index.ts",
+ "tier": "X",
+ "env": [
+ "GEMINI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "reason": "dropped-provider",
+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"
+ },
+ {
+ "id": "ts/google/experimental.mcp",
+ "lang": "ts",
+ "pkg": "google-example",
+ "file": "ts/examples/google/src/experimental.mcp.ts",
+ "tier": "X",
+ "env": [
+ "GEMINI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "reason": "dropped-provider",
+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"
+ },
+ {
+ "id": "ts/json-schema-to-zod/index",
+ "lang": "ts",
+ "pkg": "json-schema-to-zod-example",
+ "file": "ts/examples/json-schema-to-zod/src/index.ts",
+ "tier": "1",
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/langchain/index",
+ "lang": "ts",
+ "pkg": "langchain-example",
+ "file": "ts/examples/langchain/src/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/langchain/openai",
+ "lang": "ts",
+ "pkg": "langchain-example",
+ "file": "ts/examples/langchain/src/openai.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 300
+ },
+ {
+ "id": "ts/llamaindex/index",
+ "lang": "ts",
+ "pkg": "llamaindex-example",
+ "file": "ts/examples/llamaindex/src/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/mastra/index",
+ "lang": "ts",
+ "pkg": "mastra-example",
+ "file": "ts/examples/mastra/src/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/mastra/tool-router",
+ "lang": "ts",
+ "pkg": "mastra-example",
+ "file": "ts/examples/mastra/src/tool-router.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/mastra/smoke",
+ "lang": "ts",
+ "pkg": "mastra-example",
+ "file": "ts/examples/mastra/src/smoke.ts",
+ "tier": "1",
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/mastra/cloudflare",
+ "lang": "ts",
+ "pkg": "mastra-example",
+ "file": "ts/examples/mastra/src/cloudflare.ts",
+ "tier": "X",
+ "reason": "worker-module",
+ "notes": "covered by cf:dry-run in ts.examples.yml"
+ },
+ {
+ "id": "ts/mcp/index",
+ "lang": "ts",
+ "pkg": "mcp-example",
+ "file": "ts/examples/mcp/src/index.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/modifiers/index",
+ "lang": "ts",
+ "pkg": "modifiers-example",
+ "file": "ts/examples/modifiers/src/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/index",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/tool-router",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/tool-router.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/smoke",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/smoke.ts",
+ "tier": "1",
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/openai/chat-completions",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/chat-completions.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/tools",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/tools.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "googledrive"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/cloudflare",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/cloudflare.ts",
+ "tier": "X",
+ "reason": "worker-module",
+ "notes": "covered by cf:dry-run in ts.examples.yml"
+ },
+ {
+ "id": "ts/openai/agents-api",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/agents-api/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180,
+ "llmMock": false,
+ "notes": "@openai/agents rejects aimock responses-api output; live canary only."
+ },
+ {
+ "id": "ts/openai/agents-api-mcp",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/agents-api/experiment.mcp.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/agents-api-tool-router",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/agents-api/experiment.tool-router.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180,
+ "llmMock": false,
+ "notes": "@openai/agents rejects aimock responses-api output; live canary only."
+ },
+ {
+ "id": "ts/openai/responses-api",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/responses-api/index.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/openai/responses-api-mcp",
+ "lang": "ts",
+ "pkg": "openai-example",
+ "file": "ts/examples/openai/src/responses-api/experiment.mcp.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/session-management/index",
+ "lang": "ts",
+ "pkg": "session-management-example",
+ "file": "ts/examples/session-management/src/index.ts",
+ "tier": "1",
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/tool-router/index",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/index.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/tool-router/authorize",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/authorize.ts",
+ "tier": "3",
+ "readiness": "Visit this URL to authorize: https?://",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "ts/tool-router/claude-agent-sdk",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/claude-agent-sdk.ts",
+ "tier": "2",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 300,
+ "notes": "needs @anthropic-ai/claude-code CLI on PATH"
+ },
+ {
+ "id": "ts/tool-router/custom-tools",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/custom-tools.ts",
+ "tier": "2",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/tool-router/custom-tools-agent",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/custom-tools-agent.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/tool-router/custom-tools-e2e",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/custom-tools-e2e.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "weathermap",
+ "gmail"
+ ],
+ "timeoutSec": 240
+ },
+ {
+ "id": "ts/tool-router/direct-tools-preset",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/direct-tools-preset.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/tool-router/langchain",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/langchain.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "ts/tool-router/mcp",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/mcp.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/tool-router/multi-account",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/multi-account.ts",
+ "tier": "3",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "readiness": "Visit this URL to authorize: https?://",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 120,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "ts/tool-router/openai-agents",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/openai-agents.ts",
+ "tier": "X",
+ "reason": "stdin-repl"
+ },
+ {
+ "id": "ts/tool-router/preload",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/preload.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/tool-router/session-update",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/session-update.ts",
+ "tier": "1",
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/tool-router/toolkits",
+ "lang": "ts",
+ "pkg": "tool-router-example",
+ "file": "ts/examples/tool-router/src/toolkits.ts",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/toolkits/index",
+ "lang": "ts",
+ "pkg": "toolkits-example",
+ "file": "ts/examples/toolkits/src/index.ts",
+ "tier": "3",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "readiness": "authorize the toolkit: https?://",
+ "toolkits": [
+ "github"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/toolkits/toolkits",
+ "lang": "ts",
+ "pkg": "toolkits-example",
+ "file": "ts/examples/toolkits/src/toolkits.ts",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/toolkits/auth-fields",
+ "lang": "ts",
+ "pkg": "toolkits-example",
+ "file": "ts/examples/toolkits/src/auth-fields.ts",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/tools/index",
+ "lang": "ts",
+ "pkg": "tools-example",
+ "file": "ts/examples/tools/src/index.ts",
+ "tier": "2",
+ "toolkits": [
+ "googledrive"
+ ],
+ "timeoutSec": 180,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "ts/tools/ai",
+ "lang": "ts",
+ "pkg": "tools-example",
+ "file": "ts/examples/tools/src/ai.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "googledrive"
+ ],
+ "timeoutSec": 180,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "ts/tools/tool-conversion",
+ "lang": "ts",
+ "pkg": "tools-example",
+ "file": "ts/examples/tools/src/tool-conversion.ts",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/triggers/index",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/index.ts",
+ "tier": "3",
+ "readiness": "Subscribed to trigger events",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/triggers/create",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/create.ts",
+ "tier": "2",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "serial": true,
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/triggers/subscribe",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/subscribe.ts",
+ "tier": "3",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "readiness": "Subscribed to triggers",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/triggers/test",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/test.ts",
+ "tier": "2",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 120,
+ "notes": "triage: entry not yet characterized"
+ },
+ {
+ "id": "ts/triggers/trigger-types",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/trigger-types.ts",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "ts/triggers/webhook-server",
+ "lang": "ts",
+ "pkg": "triggers-example",
+ "file": "ts/examples/triggers/src/webhook-server.ts",
+ "tier": "X",
+ "reason": "external-tunnel"
+ },
+ {
+ "id": "ts/vercel/ai",
+ "lang": "ts",
+ "pkg": "vercel-example",
+ "file": "ts/examples/vercel/src/ai.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/vercel/stream",
+ "lang": "ts",
+ "pkg": "vercel-example",
+ "file": "ts/examples/vercel/src/stream.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/vercel/tool-loop-agent",
+ "lang": "ts",
+ "pkg": "vercel-example",
+ "file": "ts/examples/vercel/src/tool-loop-agent.ts",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 240
+ },
+ {
+ "id": "ts/vercel/tool-router-ai",
+ "lang": "ts",
+ "pkg": "vercel-example",
+ "file": "ts/examples/vercel/src/tool-router-ai.ts",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180
+ },
+ {
+ "id": "ts/versioning/index",
+ "lang": "ts",
+ "pkg": "versioning-example",
+ "file": "ts/examples/versioning/src/index.ts",
+ "tier": "1",
+ "toolkits": [
+ "gmail",
+ "googlecalendar"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/tools",
+ "lang": "py",
+ "file": "python/examples/tools.py",
+ "tier": "2",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GITHUB_CONNECTED_ACCOUNT_ID"
+ ],
+ "toolkits": [
+ "github"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/toolkits",
+ "lang": "py",
+ "file": "python/examples/toolkits.py",
+ "tier": "2",
+ "toolkits": [
+ "github"
+ ],
+ "serial": true,
+ "timeoutSec": 120,
+ "notes": "authorize() initiates OAuth state"
+ },
+ {
+ "id": "py/auth_configs",
+ "lang": "py",
+ "file": "python/examples/auth_configs.py",
+ "tier": "2",
+ "toolkits": [],
+ "serial": true,
+ "timeoutSec": 120,
+ "notes": "creates/updates/deletes org auth configs"
+ },
+ {
+ "id": "py/connected_accounts",
+ "lang": "py",
+ "file": "python/examples/connected_accounts.py",
+ "tier": "3",
+ "readiness": "Visit this URL to authorize: https?://",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_APIKEY_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_APIKEY_PLACEHOLDER",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "py/triggers",
+ "lang": "py",
+ "file": "python/examples/triggers.py",
+ "tier": "3",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "readiness": "Subscribed|[Ll]istening for events",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 300
+ },
+ {
+ "id": "py/modifiers",
+ "lang": "py",
+ "file": "python/examples/modifiers.py",
+ "tier": "X",
+ "env": [
+ "NOTION_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews",
+ "notion"
+ ],
+ "reason": "dropped-provider",
+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"
+ },
+ {
+ "id": "py/mcp",
+ "lang": "py",
+ "file": "python/examples/mcp_example.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "ids": [
+ "COMPOSIO_EXAMPLES_SLACK_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "toolkits": [
+ "slack"
+ ],
+ "timeoutSec": 180,
+ "pyWith": [
+ "langchain-mcp-adapters",
+ "langgraph",
+ "langchain",
+ "langchain-openai"
+ ]
+ },
+ {
+ "id": "py/fastapi_app",
+ "lang": "py",
+ "file": "python/examples/fastapi_app.py",
+ "tier": "3",
+ "readiness": "Uvicorn running|Application startup",
+ "ids": [
+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ],
+ "pyWith": [
+ "fastapi",
+ "uvicorn"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/custom_tools_agent_test",
+ "lang": "py",
+ "file": "python/examples/custom_tools_agent_test.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 240,
+ "pyWith": [
+ "openai-agents>=0.19"
+ ]
+ },
+ {
+ "id": "py/experimental_tool_router",
+ "lang": "py",
+ "file": "python/examples/experimental_tool_router_example.py",
+ "tier": "3",
+ "readiness": "Redirect URL: https?://",
+ "toolkits": [
+ "github"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/experimental_tool_router_advanced",
+ "lang": "py",
+ "file": "python/examples/experimental_tool_router_advanced_example.py",
+ "tier": "1",
+ "toolkits": [
+ "github",
+ "slack",
+ "linear",
+ "jira"
+ ],
+ "timeoutSec": 120,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID",
+ "COMPOSIO_EXAMPLES_GITHUB_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_SLACK_AUTH_CONFIG_ID",
+ "COMPOSIO_EXAMPLES_GITHUB_CONNECTED_ACCOUNT_ID",
+ "COMPOSIO_EXAMPLES_SLACK_CONNECTED_ACCOUNT_ID"
+ ]
+ },
+ {
+ "id": "py/tool_router/authorize",
+ "lang": "py",
+ "file": "python/examples/tool_router/authorize.py",
+ "tier": "3",
+ "readiness": "Redirect URL: https?://",
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 90
+ },
+ {
+ "id": "py/tool_router/claude_agent",
+ "lang": "py",
+ "file": "python/examples/tool_router/claude_agent.py",
+ "tier": "2",
+ "env": [
+ "ANTHROPIC_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "pyWith": [
+ "claude-agent-sdk",
+ "composio-claude-agent-sdk",
+ "./python"
+ ],
+ "timeoutSec": 300,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "py/tool_router/crewai_agent",
+ "lang": "py",
+ "file": "python/examples/tool_router/crewai_agent.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "pyWith": [
+ "crewai>=1.15"
+ ],
+ "timeoutSec": 300,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "py/tool_router/direct_tools_preset",
+ "lang": "py",
+ "file": "python/examples/tool_router/direct_tools_preset.py",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180,
+ "pyWith": [
+ "openai-agents>=0.19"
+ ]
+ },
+ {
+ "id": "py/tool_router/preload",
+ "lang": "py",
+ "file": "python/examples/tool_router/preload.py",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "hackernews"
+ ],
+ "timeoutSec": 180,
+ "pyWith": [
+ "openai-agents>=0.19"
+ ]
+ },
+ {
+ "id": "py/tool_router/files",
+ "lang": "py",
+ "file": "python/examples/tool_router/files.py",
+ "tier": "1",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [],
+ "timeoutSec": 180
+ },
+ {
+ "id": "py/tool_router/langchain_agent",
+ "lang": "py",
+ "file": "python/examples/tool_router/langchain_agent.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "pyWith": [
+ "langchain",
+ "langchain-openai",
+ "langchain-mcp-adapters"
+ ],
+ "timeoutSec": 240,
+ "ids": [
+ "COMPOSIO_EXAMPLES_USER_ID"
+ ]
+ },
+ {
+ "id": "py/tool_router/openai_agents",
+ "lang": "py",
+ "file": "python/examples/tool_router/openai_agents.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "pyWith": [
+ "openai-agents>=0.19"
+ ],
+ "timeoutSec": 240
+ },
+ {
+ "id": "py/tool_router/tool_router_mcp",
+ "lang": "py",
+ "file": "python/examples/tool_router/tool_router_mcp.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "github",
+ "gmail"
+ ],
+ "timeoutSec": 240,
+ "pyWith": [
+ "openai-agents>=0.19"
+ ]
+ },
+ {
+ "id": "py/tool_router/session_update",
+ "lang": "py",
+ "file": "python/examples/tool_router/session_update.py",
+ "tier": "1",
+ "toolkits": [
+ "gmail",
+ "github"
+ ],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/tool_router/toolkits",
+ "lang": "py",
+ "file": "python/examples/tool_router/toolkits.py",
+ "tier": "1",
+ "toolkits": [],
+ "timeoutSec": 120
+ },
+ {
+ "id": "py/tool_router/tools",
+ "lang": "py",
+ "file": "python/examples/tool_router/tools.py",
+ "tier": "2",
+ "env": [
+ "OPENAI_API_KEY"
+ ],
+ "toolkits": [
+ "gmail"
+ ],
+ "timeoutSec": 180,
+ "pyWith": [
+ "openai-agents>=0.19"
+ ]
+ }
+ ]
+}
diff --git a/harness/fixtures/always-fails.ts b/harness/fixtures/always-fails.ts
new file mode 100644
index 000000000..9e49d7085
--- /dev/null
+++ b/harness/fixtures/always-fails.ts
@@ -0,0 +1,3 @@
+// Known-bad selftest fixture: must exit non-zero.
+console.error('fixture: failing on purpose');
+process.exit(1);
diff --git a/harness/fixtures/always_fails.py b/harness/fixtures/always_fails.py
new file mode 100644
index 000000000..21b115275
--- /dev/null
+++ b/harness/fixtures/always_fails.py
@@ -0,0 +1,6 @@
+"""Known-bad selftest fixture: must exit non-zero."""
+
+import sys
+
+print("fixture: failing on purpose", file=sys.stderr)
+sys.exit(1)
diff --git a/harness/fixtures/trace-check.ts b/harness/fixtures/trace-check.ts
new file mode 100644
index 000000000..c6a47369b
--- /dev/null
+++ b/harness/fixtures/trace-check.ts
@@ -0,0 +1,7 @@
+// Selftest fixture: makes one unauthenticated request against the backend so
+// the fetch shim must record a non-2xx composio line in COMPOSIO_TRACE_FILE.
+const base = process.env.COMPOSIO_BASE_URL ?? 'https://staging-backend.composio.dev';
+const res = await fetch(`${base}/api/v3/toolkits`, {
+ headers: { 'x-api-key': 'selftest-invalid-key' },
+});
+console.log(`trace-check: status ${res.status}`);
diff --git a/harness/fixtures/trace_check.py b/harness/fixtures/trace_check.py
new file mode 100644
index 000000000..c29a4f64a
--- /dev/null
+++ b/harness/fixtures/trace_check.py
@@ -0,0 +1,11 @@
+"""Selftest fixture: one unauthenticated httpx request against the backend so
+the sitecustomize shim must record a non-2xx composio line."""
+
+import os
+
+import httpx
+
+base = os.environ.get("COMPOSIO_BASE_URL", "https://staging-backend.composio.dev")
+with httpx.Client() as client:
+ response = client.get(f"{base}/api/v3/toolkits", headers={"x-api-key": "selftest-invalid-key"})
+print(f"trace-check: status {response.status_code}")
diff --git a/harness/llm-mock/aimock.json b/harness/llm-mock/aimock.json
new file mode 100644
index 000000000..fc516d466
--- /dev/null
+++ b/harness/llm-mock/aimock.json
@@ -0,0 +1,5 @@
+{
+ "llm": {
+ "fixtures": "./fixtures"
+ }
+}
diff --git a/harness/llm-mock/fixtures/default.json b/harness/llm-mock/fixtures/default.json
new file mode 100644
index 000000000..7e6e813c4
--- /dev/null
+++ b/harness/llm-mock/fixtures/default.json
@@ -0,0 +1,29 @@
+{
+ "$comment": "Deterministic LLM fixtures for --llm mock sweeps. One file so fixture order is explicit: specific matches first, catch-alls last (first match wins). Tool-call fixtures guard with hasToolResult:false so the follow-up turn falls through to the wrap-up fixture. Composio tool execution stays live; only the model is mocked.",
+ "fixtures": [
+ {
+ "match": { "userMessage": "HackerNews user", "hasToolResult": false },
+ "response": {
+ "toolCalls": [
+ { "name": "HACKERNEWS_GET_USER", "arguments": { "username": "pg" }, "id": "call_hn_user" }
+ ]
+ }
+ },
+ {
+ "match": { "userMessage": "HackerNews", "hasToolResult": false },
+ "response": {
+ "toolCalls": [
+ { "name": "HACKERNEWS_GET_TOP_STORIES", "arguments": { "size": 3 }, "id": "call_hn_top" }
+ ]
+ }
+ },
+ {
+ "match": { "hasToolResult": true },
+ "response": { "content": "Mock model wrap-up: summarized the tool result deterministically." }
+ },
+ {
+ "match": {},
+ "response": { "content": "Mock model response: no tool needed for this prompt." }
+ }
+ ]
+}
diff --git a/harness/parity.mjs b/harness/parity.mjs
new file mode 100755
index 000000000..89b73dda8
--- /dev/null
+++ b/harness/parity.mjs
@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+// Trace-parity comparator: node harness/parity.mjs <baseline-run-dir> <candidate-run-dir>
+// Per entry green in BOTH runs: the sets of distinct (method, path-template)
+// pairs must be equal, ignoring pairs allowlisted in parity-variance.json.
+// Prints a JSON report; exit 0 = all parities hold, 1 = mismatches.
+
+import { readFileSync, readdirSync, existsSync } from 'node:fs';
+import { join, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const [baseDir, candDir] = process.argv.slice(2);
+if (!baseDir || !candDir) {
+ console.error('usage: parity.mjs <baseline-run-dir> <candidate-run-dir>');
+ process.exit(2);
+}
+
+const variance = JSON.parse(readFileSync(join(ROOT, 'parity-variance.json'), 'utf8'));
+const ignored = new Set((variance.entries ?? []).map((e) => e.pair));
+
+const loadRun = (dir) => {
+ const results = readFileSync(join(dir, 'results.jsonl'), 'utf8')
+ .trim()
+ .split('\n')
+ .filter(Boolean)
+ .map((l) => JSON.parse(l));
+ const byId = new Map();
+ for (const r of results) byId.set(r.id, r);
+ return byId;
+};
+
+const tracePairs = (dir, id) => {
+ const file = join(dir, 'traces', `${id.replace(/[/]/g, '__')}.jsonl`);
+ const pairs = new Set();
+ if (!existsSync(file)) return pairs;
+ for (const line of readFileSync(file, 'utf8').trim().split('\n').filter(Boolean)) {
+ const rec = JSON.parse(line);
+ if (rec.m && rec.p) pairs.add(`${rec.m} ${rec.p}`);
+ }
+ return pairs;
+};
+
+const baseline = loadRun(baseDir);
+const candidate = loadRun(candDir);
+
+const report = [];
+const ids = new Set([...baseline.keys(), ...candidate.keys()]);
+for (const id of [...ids].sort()) {
+ const b = baseline.get(id);
+ const c = candidate.get(id);
+ if (!b || !c) {
+ report.push({
+ id,
+ parity: false,
+ reason: `missing ${b ? 'candidate' : 'baseline'} result`,
+ });
+ continue;
+ }
+ if (b.status !== 'green' || c.status !== 'green') {
+ report.push({ id, parity: false, reason: `status baseline=${b.status} candidate=${c.status}` });
+ continue;
+ }
+ const bp = tracePairs(baseDir, id);
+ const cp = tracePairs(candDir, id);
+ const onlyBaseline = [...bp].filter((p) => !cp.has(p) && !ignored.has(p));
+ const onlyCandidate = [...cp].filter((p) => !bp.has(p) && !ignored.has(p));
+ const match = onlyBaseline.length === 0 && onlyCandidate.length === 0;
+ report.push({ id, parity: match, ...(match ? {} : { onlyBaseline, onlyCandidate }) });
+}
+
+const green = report.filter((r) => r.parity).length;
+console.log(JSON.stringify({ compared: report.length, parityGreen: green, ignoredPairs: ignored.size, entries: report }, null, 2));
+process.exit(report.length > 0 && report.every((r) => r.parity) ? 0 : 1);
diff --git a/harness/run.mjs b/harness/run.mjs
new file mode 100755
index 000000000..bf159fa6a
--- /dev/null
+++ b/harness/run.mjs
@@ -0,0 +1,545 @@
+#!/usr/bin/env node
+// Examples runner: sweeps the entrypoints in examples-manifest.json against
+// the live backend, recording per-entry results and Composio traces.
+//
+// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--llm live|mock]
+// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--sample N] [--seed S]
+// node harness/run.mjs selftest
+//
+// Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.
+
+import { spawn } from 'node:child_process';
+import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';
+import { dirname, join, relative, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { requireStagingBaseUrl, STAGING_BASE_URL } from './staging-backend.mjs';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const ARTIFACTS = join(ROOT, '.artifacts', 'examples-parity');
+const STAINLESS_TS_VERSION = '0.1.0-alpha.76';
+const AIMOCK_VERSION = '1.38.0';
+const AIMOCK_PORT = Number(process.env.AIMOCK_PORT ?? 4010);
+
+// Set by cmdSweep when --llm mock: LLM traffic goes to a local aimock server
+// and provider keys are overridden so a mock sweep can never spend tokens.
+let LLM_MOCK = false;
+
+const args = process.argv.slice(2);
+const cmd = args[0];
+const opt = (name, fallback) => {
+ const i = args.indexOf(`--${name}`);
+ return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : fallback;
+};
+
+class HarnessError extends Error {
+ constructor(message, exitCode = 2) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+}
+
+const fail = (msg, code = 2) => {
+ throw new HarnessError(msg, code);
+};
+
+const loadManifest = () => {
+ const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));
+ for (const e of manifest.entries) {
+ if (!e.id || !e.lang || !e.file || !e.tier) fail(`manifest entry missing required fields: ${JSON.stringify(e)}`);
+ if (e.lang === 'ts' && e.tier !== 'X' && !e.pkg) fail(`ts entry ${e.id} missing pkg`);
+ if (e.tier === '3' && !e.readiness) fail(`tier-3 entry ${e.id} missing readiness regex`);
+ }
+ return manifest.entries;
+};
+
+const selectEntries = () => {
+ const lang = opt('lang');
+ const ids = opt('ids') ? opt('ids').split(',') : null;
+ const tiers = (opt('tiers', '1,2,3')).split(',');
+ return loadManifest().filter(
+ (e) =>
+ e.tier !== 'X' &&
+ tiers.includes(e.tier) &&
+ (!lang || e.lang === lang) &&
+ (!ids || ids.includes(e.id))
+ );
+};
+
+const baseUrl = () => {
+ try {
+ return requireStagingBaseUrl();
+ } catch (error) {
+ fail(error instanceof Error ? error.message : String(error));
+ }
+};
+
+// Deterministic PRNG so --seed reproduces a negative-control sample.
+const mulberry32 = (a) => () => {
+ a |= 0; a = (a + 0x6d2b79f5) | 0;
+ let t = Math.imul(a ^ (a >>> 15), 1 | a);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+};
+
+const runId = () => new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) + '-' + Math.random().toString(36).slice(2, 8);
+
+/** Spawn one entry; resolve {exit, readinessMatched, timedOut, durationMs}. */
+const runEntry = (entry, env, { garbage = false } = {}) =>
+ new Promise((resolveRun) => {
+ const isTs = entry.lang === 'ts';
+ let cmdline;
+ if (isTs && entry.script) {
+ cmdline = ['pnpm', '--filter', entry.pkg, 'run', entry.script];
+ } else if (isTs) {
+ // pnpm --filter runs with the example package as cwd; derive the package
+ // dir from the entry file path (ts/examples/<dir>/src/...).
+ const pkgDir = entry.file.split('/').slice(0, 3).join('/');
+ const srcRel = relative(join(ROOT, pkgDir), join(ROOT, entry.file));
+ cmdline = ['pnpm', '--filter', entry.pkg, 'exec', 'tsx', srcRel];
+ } else {
+ cmdline = ['uv', 'run', '--project', 'python'];
+ for (const dep of entry.pyWith ?? []) cmdline.push('--with', dep);
+ cmdline.push('python', join(ROOT, entry.file));
+ }
+
+ const child = spawn(cmdline[0], cmdline.slice(1), {
+ cwd: ROOT,
+ env,
+ detached: true,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+
+ const started = Date.now();
+ const timeoutMs = (entry.timeoutSec ?? 120) * 1000;
+ let readinessMatched = false;
+ let timedOut = false;
+ let settled = false;
+ let outputTail = '';
+ const readiness = entry.tier === '3' && entry.readiness ? new RegExp(entry.readiness) : null;
+
+ const killTree = () => {
+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
+ setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 10_000).unref();
+ };
+
+ const onChunk = (chunk) => {
+ const text = chunk.toString();
+ outputTail = (outputTail + text).slice(-4000);
+ if (readiness && !readinessMatched && readiness.test(outputTail)) {
+ readinessMatched = true;
+ // Grace period so the in-flight backend call lands in the trace, then stop the listener.
+ setTimeout(killTree, 3000).unref();
+ }
+ };
+ child.stdout.on('data', onChunk);
+ child.stderr.on('data', onChunk);
+
+ const timer = setTimeout(() => {
+ timedOut = true;
+ killTree();
+ }, timeoutMs);
+
+ child.on('exit', (code, signal) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ resolveRun({
+ exit: code ?? (signal ? 128 : 1),
+ readinessMatched,
+ timedOut,
+ durationMs: Date.now() - started,
+ outputTail: garbage ? undefined : outputTail.slice(-1500),
+ });
+ });
+ child.on('error', () => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ resolveRun({ exit: 127, readinessMatched: false, timedOut: false, durationMs: Date.now() - started });
+ });
+ });
+
+const isGreen = (entry, r) => {
+ if (entry.tier === '3') return r.readinessMatched || (!r.timedOut && r.exit === 0);
+ return !r.timedOut && r.exit === 0;
+};
+
+const buildEnv = (entry, runDir, { garbage = false } = {}) => {
+ const traceFile = join(runDir, 'traces', `${entry.id.replace(/[/]/g, '__')}.jsonl`);
+ mkdirSync(dirname(traceFile), { recursive: true });
+ const env = { ...process.env };
+ env.COMPOSIO_BASE_URL = baseUrl();
+ env.COMPOSIO_TRACE_FILE = traceFile;
+ env.COMPOSIO_LOG_LEVEL ??= 'error';
+ // The outbound-email denylist is fixed by the shims; never overridable per run.
+ delete env.COMPOSIO_TOOL_DENYLIST;
+ if (entry.lang === 'ts') {
+ env.NODE_OPTIONS = `${env.NODE_OPTIONS ?? ''} --import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}`.trim();
+ } else {
+ env.PYTHONPATH = `${join(ROOT, 'harness', 'trace-py')}${env.PYTHONPATH ? `:${env.PYTHONPATH}` : ''}`;
+ }
+ if (garbage) {
+ env.COMPOSIO_API_KEY = 'nc-invalid-0000000000000000';
+ env.OPENAI_API_KEY = 'nc-invalid';
+ env.ANTHROPIC_API_KEY = 'nc-invalid';
+ env.GEMINI_API_KEY = 'nc-invalid';
+ env.NOTION_API_KEY = 'nc-invalid';
+ } else if (LLM_MOCK) {
+ const mock = `http://127.0.0.1:${AIMOCK_PORT}`;
+ env.OPENAI_BASE_URL = `${mock}/v1`;
+ // langchain/litellm read the older variable name.
+ env.OPENAI_API_BASE = `${mock}/v1`;
+ env.ANTHROPIC_BASE_URL = mock;
+ env.OPENAI_API_KEY = 'aimock';
+ env.ANTHROPIC_API_KEY = 'aimock';
+ }
+ return { env, traceFile };
+};
+
+const LLM_KEYS = new Set(['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GEMINI_API_KEY']);
+
+const missingEnv = (entry) => {
+ const wanted = [...(entry.env ?? []), ...(entry.ids ?? [])].filter(
+ (k) => !(LLM_MOCK && LLM_KEYS.has(k))
+ );
+ if (entry.backend !== false) wanted.unshift('COMPOSIO_API_KEY');
+ return wanted.filter((k) => !process.env[k]);
+};
+
+/** Run entries with bounded concurrency; serial entries run last, alone. */
+const runPool = async (entries, runDir, { garbage = false, concurrency = 4 } = {}) => {
+ const parallel = entries.filter((e) => !e.serial);
+ const serial = entries.filter((e) => e.serial);
+ const results = [];
+ const worker = async (queue) => {
+ for (;;) {
+ const entry = queue.shift();
+ if (!entry) return;
+ if (LLM_MOCK && entry.llmMock === false) {
+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: 'llm-mock-unsupported' });
+ console.log(` ~ ${entry.id} skipped (llm-mock-unsupported)`);
+ continue;
+ }
+ const missing = garbage ? [] : missingEnv(entry);
+ if (missing.length > 0) {
+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: `missing-env:${missing.join(',')}` });
+ console.log(` ~ ${entry.id} skipped (${missing.join(',')})`);
+ continue;
+ }
+ const { env, traceFile } = buildEnv(entry, runDir, { garbage });
+ const r = await runEntry(entry, env, { garbage });
+ // An entry that tripped the outbound-email guard is red no matter how it
+ // exited: catching the refusal and continuing is not coverage.
+ const blocked =
+ existsSync(traceFile) && readFileSync(traceFile, 'utf8').includes('"s":"BLOCKED"');
+ const green = !blocked && isGreen(entry, r);
+ results.push({
+ id: entry.id,
+ tier: entry.tier,
+ status: green ? 'green' : 'red',
+ ...(blocked ? { reason: 'blocked-tool' } : {}),
+ exit: r.exit,
+ timedOut: r.timedOut,
+ readinessMatched: r.readinessMatched,
+ durationMs: r.durationMs,
+ traceFile: relative(ROOT, traceFile),
+ outputTail: green ? undefined : r.outputTail,
+ });
+ console.log(` ${green ? '✓' : '✗'} ${entry.id} (${(r.durationMs / 1000).toFixed(1)}s${r.timedOut ? ', timeout' : ''})`);
+ }
+ };
+ const queue = [...parallel];
+ await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(queue.length, 1)) }, () => worker(queue)));
+ for (const entry of serial) await worker([entry]);
+ return results;
+};
+
+// --- candidate swap (TS) --------------------------------------------------
+
+const WORKSPACE_YAML = join(ROOT, 'pnpm-workspace.yaml');
+const LOCKFILE = join(ROOT, 'pnpm-lock.yaml');
+
+const sh = (cmdline, opts = {}) =>
+ new Promise((resolveSh, rejectSh) => {
+ const child = spawn(cmdline[0], cmdline.slice(1), { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'], ...opts });
+ let out = '';
+ child.stdout.on('data', (c) => (out += c));
+ child.stderr.on('data', (c) => (out += c));
+ child.on('exit', (code) => (code === 0 ? resolveSh(out) : rejectSh(new Error(`${cmdline.join(' ')} → exit ${code}\n${out.slice(-2000)}`))));
+ });
+
+const resolvedTsClientVersion = async () => {
+ const out = await sh(['node', '-e', "console.log(JSON.parse(require('fs').readFileSync(require.resolve('@composio/client/package.json', {paths:[require('path').join(process.cwd(),'ts/packages/core')]}), 'utf8')).version)"]);
+ return out.trim().split('\n').pop();
+};
+
+const applyTsCandidate = async (tarball) => {
+ if (!existsSync(tarball)) fail(`COMPOSIO_CLIENT_TARBALL not found: ${tarball}`);
+ const yaml = readFileSync(WORKSPACE_YAML, 'utf8');
+ if (/^overrides:$/m.test(yaml)) {
+ // Merge into the repo's existing overrides block (e.g. security pins);
+ // restoreTsBaseline() writes back the exact pre-run snapshot either way.
+ writeFileSync(WORKSPACE_YAML, yaml.replace(/^overrides:$/m, `overrides:\n '@composio/client': file:${tarball}`));
+ } else {
+ writeFileSync(WORKSPACE_YAML, `${yaml}\noverrides:\n '@composio/client': file:${tarball}\n`);
+ }
+ await sh(['pnpm', 'install', '--no-frozen-lockfile']);
+ const version = await resolvedTsClientVersion();
+ if (version === STAINLESS_TS_VERSION) {
+ fail(`override did not take effect: @composio/client still resolves to ${version}`);
+ }
+ console.log(`candidate @composio/client resolved: ${version}`);
+ await sh(['pnpm', 'run', 'build:packages']);
+};
+
+const snapshotFiles = (paths) => new Map(paths.map((path) => [path, readFileSync(path)]));
+
+const restoreFiles = (snapshot) => {
+ for (const [path, contents] of snapshot) writeFileSync(path, contents);
+};
+
+const snapshotTsBaseline = () => snapshotFiles([WORKSPACE_YAML, LOCKFILE]);
+
+const restoreTsBaseline = async (snapshot) => {
+ restoreFiles(snapshot);
+ await sh(['pnpm', 'install', '--frozen-lockfile']);
+};
+
+const verifyPyCandidate = async (wheel) => {
+ if (!existsSync(wheel)) fail(`COMPOSIO_CLIENT_WHEEL not found: ${wheel}`);
+ const out = await sh(['uv', 'run', '--project', 'python', '--with', wheel, 'python', '-c', 'import composio_client, importlib.metadata; print(importlib.metadata.version("composio-client"))']);
+ const version = out.trim().split('\n').pop();
+ if (!version.startsWith('2.')) {
+ fail(`python candidate did not take effect (resolved ${version}); Stage B (SDK migration) must land first`);
+ }
+ return version;
+};
+
+// --- llm mock ---------------------------------------------------------------
+
+/** Start aimock serving harness/llm-mock/fixtures; resolve once it answers. */
+const startLlmMock = async (runDir) => {
+ const mockDir = join(ROOT, 'harness', 'llm-mock');
+ const logFile = join(runDir, 'aimock.log');
+ const child = spawn(
+ 'npx',
+ ['-y', `@copilotkit/aimock@${AIMOCK_VERSION}`, '-c', 'aimock.json', '-p', String(AIMOCK_PORT), '-h', '127.0.0.1'],
+ { cwd: mockDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }
+ );
+ child.stdout.on('data', (c) => appendFileSync(logFile, c));
+ child.stderr.on('data', (c) => appendFileSync(logFile, c));
+
+ // First run may download the package; allow a generous readiness window.
+ const deadline = Date.now() + 120_000;
+ for (;;) {
+ try {
+ const res = await fetch(`http://127.0.0.1:${AIMOCK_PORT}/v1/models`);
+ if (res.ok) break;
+ } catch { /* not up yet */ }
+ if (Date.now() > deadline) {
+ try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ }
+ fail(`aimock did not become ready on :${AIMOCK_PORT} (see ${relative(ROOT, logFile)})`);
+ }
+ await new Promise((r) => setTimeout(r, 500));
+ }
+ console.log(`aimock ${AIMOCK_VERSION} ready on :${AIMOCK_PORT}`);
+ return () => {
+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
+ };
+};
+
+// --- commands ---------------------------------------------------------------
+
+const cmdSweep = async () => {
+ const client = opt('client', 'baseline');
+ if (!['baseline', 'candidate'].includes(client)) fail(`--client must be baseline|candidate`);
+ const llm = opt('llm', 'live');
+ if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);
+ LLM_MOCK = llm === 'mock';
+ const entries = selectEntries();
+ if (entries.length === 0) fail('no entries selected');
+ if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);
+
+ const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;
+ const runDir = join(ARTIFACTS, id);
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
+
+ let tsBaselineSnapshot;
+ let stopLlmMock;
+ const pyWheel = process.env.COMPOSIO_CLIENT_WHEEL;
+ try {
+ if (LLM_MOCK) stopLlmMock = await startLlmMock(runDir);
+ if (client === 'candidate') {
+ const wantTs = entries.some((e) => e.lang === 'ts');
+ const wantPy = entries.some((e) => e.lang === 'py');
+ if (wantTs) {
+ if (!process.env.COMPOSIO_CLIENT_TARBALL) fail('candidate sweep with ts entries needs COMPOSIO_CLIENT_TARBALL');
+ tsBaselineSnapshot = snapshotTsBaseline();
+ await applyTsCandidate(process.env.COMPOSIO_CLIENT_TARBALL);
+ }
+ if (wantPy) {
+ if (!pyWheel) fail('candidate sweep with py entries needs COMPOSIO_CLIENT_WHEEL');
+ const v = await verifyPyCandidate(pyWheel);
+ console.log(`candidate composio-client (py) resolved: ${v}`);
+ for (const e of entries) if (e.lang === 'py') e.pyWith = [...(e.pyWith ?? []), pyWheel];
+ }
+ }
+
+ console.log(`sweep ${id}: ${entries.length} entries → ${runDir}`);
+ const results = await runPool(entries, runDir, { concurrency: Number(opt('concurrency', '4')) });
+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
+ const counts = {
+ green: results.filter((r) => r.status === 'green').length,
+ red: results.filter((r) => r.status === 'red').length,
+ skipped: results.filter((r) => r.status === 'skipped').length,
+ };
+ writeFileSync(
+ join(runDir, 'summary.json'),
+ JSON.stringify({ runId: id, client, llm, baseUrl: baseUrl(), counts, startedAt: new Date().toISOString() }, null, 2)
+ );
+ console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);
+ process.exitCode = counts.red > 0 ? 1 : 0;
+ } finally {
+ if (stopLlmMock) stopLlmMock();
+ if (tsBaselineSnapshot) await restoreTsBaseline(tsBaselineSnapshot);
+ }
+};
+
+const cmdNeg = async () => {
+ let entries = selectEntries().filter((e) => e.backend !== false);
+ const sample = opt('sample');
+ if (sample) {
+ const rand = mulberry32(Number(opt('seed', '42')));
+ entries = [...entries].sort(() => rand() - 0.5).slice(0, Number(sample));
+ }
+ const id = `${runId()}-neg`;
+ const runDir = join(ARTIFACTS, id);
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
+ console.log(`negative controls ${id}: ${entries.length} entries (garbage credentials; expectation: red)`);
+ const results = await runPool(entries, runDir, { garbage: true, concurrency: Number(opt('concurrency', '6')) });
+ const swallowers = results.filter((r) => r.status === 'green').map((r) => r.id);
+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
+ writeFileSync(join(runDir, 'summary.json'), JSON.stringify({ runId: id, mode: 'neg', total: results.length, swallowers }, null, 2));
+ if (swallowers.length > 0) {
+ console.log(`NEGATIVE-CONTROL FAILURES (green under garbage credentials — these swallow errors):`);
+ for (const s of swallowers) console.log(` ✗ ${s}`);
+ process.exitCode = 1;
+ } else {
+ console.log('all negative controls red, as expected');
+ }
+};
+
+const cmdSelftest = async () => {
+ const runDir = join(ARTIFACTS, `${runId()}-selftest`);
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
+ let failures = 0;
+ const check = (name, ok) => {
+ console.log(` ${ok ? '✓' : '✗'} ${name}`);
+ if (!ok) failures += 1;
+ };
+
+ // 0. backend safety: the harness accepts only the canonical staging root.
+ check('staging backend is accepted', requireStagingBaseUrl() === STAGING_BASE_URL);
+ let productionRejected = false;
+ try {
+ requireStagingBaseUrl('https://backend.composio.dev');
+ } catch {
+ productionRejected = true;
+ }
+ check('production backend is rejected', productionRejected);
+
+ let cleanupRan = false;
+ let cleanupError;
+ try {
+ try {
+ fail('expected selftest failure');
+ } finally {
+ cleanupRan = true;
+ }
+ } catch (error) {
+ cleanupError = error;
+ }
+ check(
+ 'usage failures unwind through cleanup',
+ cleanupRan && cleanupError instanceof HarnessError
+ );
+
+ const cleanupFixture = join(runDir, 'cleanup-fixture.txt');
+ writeFileSync(cleanupFixture, 'user contents');
+ const cleanupSnapshot = snapshotFiles([cleanupFixture]);
+ writeFileSync(cleanupFixture, 'candidate contents');
+ restoreFiles(cleanupSnapshot);
+ check('cleanup restores pre-run file contents', readFileSync(cleanupFixture, 'utf8') === 'user contents');
+
+ // 1. known-good: error-handling-demo (no backend, no keys).
+ const good = loadManifest().find((e) => e.id === 'ts/error-handling-demo/index');
+ const { env: goodEnv } = buildEnv(good, runDir);
+ delete goodEnv.COMPOSIO_API_KEY;
+ const goodRes = await runEntry(good, goodEnv);
+ check('known-good (error-handling-demo) exits 0', goodRes.exit === 0);
+
+ // 2. known-bad fixtures: must exit non-zero in both languages.
+ const badTsOut = await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/always-fails.ts']).then(() => 0).catch(() => 1);
+ check('known-bad ts fixture exits non-zero', badTsOut === 1);
+ const badPy = await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/always_fails.py']).then(() => 0).catch(() => 1);
+ check('known-bad py fixture exits non-zero', badPy === 1);
+
+ // 3. trace shims: an unauthenticated backend request must appear in the trace.
+ const tsTrace = join(runDir, 'traces', 'shim-ts.jsonl');
+ await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/trace-check.ts'], {
+ env: { ...process.env, COMPOSIO_TRACE_FILE: tsTrace, COMPOSIO_BASE_URL: baseUrl(), NODE_OPTIONS: `--import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}` },
+ }).catch(() => {});
+ const tsLines = existsSync(tsTrace) ? readFileSync(tsTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
+ check('ts fetch shim recorded a composio call', tsLines.some((l) => JSON.parse(l).p?.includes('/api/')));
+
+ const pyTrace = join(runDir, 'traces', 'shim-py.jsonl');
+ await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/trace_check.py'], {
+ env: { ...process.env, COMPOSIO_TRACE_FILE: pyTrace, COMPOSIO_BASE_URL: baseUrl(), PYTHONPATH: join(ROOT, 'harness', 'trace-py') },
+ }).catch(() => {});
+ const pyLines = existsSync(pyTrace) ? readFileSync(pyTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
+ check('py httpx shim recorded a composio call', pyLines.some((l) => JSON.parse(l).p?.includes('/api/')));
+
+ // 4. parity must compare at least one entry and reject missing candidates.
+ const parityBase = join(runDir, 'parity-base');
+ const parityCandidate = join(runDir, 'parity-candidate');
+ for (const dir of [parityBase, parityCandidate]) {
+ mkdirSync(join(dir, 'traces'), { recursive: true });
+ writeFileSync(join(dir, 'results.jsonl'), `${JSON.stringify({ id: 'fixture', status: 'green' })}\n`);
+ writeFileSync(
+ join(dir, 'traces', 'fixture.jsonl'),
+ `${JSON.stringify({ m: 'GET', p: '/api/v3/toolkits', s: '2xx' })}\n`
+ );
+ }
+ const parityMatches = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
+ .then(() => true)
+ .catch(() => false);
+ check('parity accepts matching runs', parityMatches);
+ writeFileSync(join(parityCandidate, 'results.jsonl'), '');
+ const missingCandidateRejected = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
+ .then(() => false)
+ .catch(() => true);
+ check('parity rejects missing candidate entries', missingCandidateRejected);
+
+ rmSync(runDir, { recursive: true, force: true });
+ console.log(failures === 0 ? 'selftest: PASS' : `selftest: FAIL (${failures})`);
+ process.exitCode = failures === 0 ? 0 : 1;
+};
+
+try {
+ switch (cmd) {
+ case 'sweep':
+ await cmdSweep();
+ break;
+ case 'neg':
+ await cmdNeg();
+ break;
+ case 'selftest':
+ await cmdSelftest();
+ break;
+ default:
+ fail('usage: run.mjs <sweep|neg|selftest> [options]');
+ }
+} catch (error) {
+ console.error(`run.mjs: ${error instanceof Error ? error.message : String(error)}`);
+ process.exitCode = error instanceof HarnessError ? error.exitCode : 1;
+}
diff --git a/harness/staging-backend.mjs b/harness/staging-backend.mjs
new file mode 100644
index 000000000..966e72469
--- /dev/null
+++ b/harness/staging-backend.mjs
@@ -0,0 +1,25 @@
+export const STAGING_BASE_URL = 'https://staging-backend.composio.dev';
+
+export const requireStagingBaseUrl = (
+ value = process.env.COMPOSIO_BASE_URL ?? STAGING_BASE_URL
+) => {
+ let url;
+ try {
+ url = new URL(value);
+ } catch {
+ throw new Error(`refusing invalid COMPOSIO_BASE_URL: ${value}`);
+ }
+
+ const isStagingRoot =
+ url.origin === STAGING_BASE_URL &&
+ url.pathname === '/' &&
+ !url.search &&
+ !url.hash &&
+ !url.username &&
+ !url.password;
+ if (!isStagingRoot) {
+ throw new Error(`refusing non-staging COMPOSIO_BASE_URL: ${value}`);
+ }
+
+ return STAGING_BASE_URL;
+};
diff --git a/harness/trace-py/sitecustomize.py b/harness/trace-py/sitecustomize.py
new file mode 100644
index 000000000..634e323ab
--- /dev/null
+++ b/harness/trace-py/sitecustomize.py
@@ -0,0 +1,108 @@
+"""Harness-owned httpx tracer, loaded via PYTHONPATH (sitecustomize hook).
+
+Examples must never reference COMPOSIO_TRACE_FILE themselves (lint-enforced).
+"""
+
+import json
+import os
+import re
+import sys
+from urllib.parse import urlsplit
+
+_TRACE = os.environ.get("COMPOSIO_TRACE_FILE")
+_STAGING_BASE_URL = "https://staging-backend.composio.dev"
+
+if _TRACE:
+ _base_url = os.environ.get("COMPOSIO_BASE_URL", _STAGING_BASE_URL)
+ _parsed_base_url = urlsplit(_base_url)
+ if not (
+ _parsed_base_url.scheme == "https"
+ and _parsed_base_url.hostname == "staging-backend.composio.dev"
+ and _parsed_base_url.path in {"", "/"}
+ and not _parsed_base_url.query
+ and not _parsed_base_url.fragment
+ and not _parsed_base_url.username
+ and not _parsed_base_url.password
+ ):
+ print(
+ f"sitecustomize: refusing non-staging COMPOSIO_BASE_URL: {_base_url}",
+ file=sys.stderr,
+ )
+ os._exit(78)
+
+ try:
+ import httpx
+
+ _LLM_HOSTS = {
+ "api.openai.com",
+ "api.anthropic.com",
+ "generativelanguage.googleapis.com",
+ }
+ _BACKEND_HOST = _parsed_base_url.hostname
+ _ID_SEG = re.compile(
+ r"^(ca_|ac_|ti_|tr_|trs_|sess_|auth_|req_|proj_|org_)[\w-]+$"
+ r"|^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
+ r"|^\d+$",
+ re.IGNORECASE,
+ )
+
+ def _template(path: str) -> str:
+ return "/".join("{id}" if _ID_SEG.match(seg) else seg for seg in path.split("/"))
+
+ def _record(obj) -> None:
+ try:
+ with open(_TRACE, "a") as f:
+ f.write(json.dumps(obj) + "\n")
+ except OSError:
+ pass
+
+ def _log(request, response) -> None:
+ host = request.url.host
+ if host == _BACKEND_HOST:
+ status = f"{response.status_code // 100}xx" if response is not None else "ERR"
+ _record({"m": request.method.upper(), "p": _template(request.url.path), "s": status})
+ elif host in _LLM_HOSTS:
+ _record({"llm": host})
+
+ # Outbound-email guard: tool executions matching the denylist are
+ # refused at the transport, never forwarded to the backend.
+ _DENY = re.compile(
+ os.environ.get(
+ "COMPOSIO_TOOL_DENYLIST",
+ r"GMAIL_SEND|GMAIL_REPLY|SEND_EMAIL|SEND_DRAFT|OUTLOOK[A-Z_]*SEND",
+ ),
+ re.IGNORECASE,
+ )
+
+ def _guard(request) -> None:
+ if request.url.host == _BACKEND_HOST and _DENY.search(request.url.path):
+ _record({"m": request.method.upper(), "p": _template(request.url.path), "s": "BLOCKED"})
+ raise RuntimeError(f"harness: outbound-email tool execution blocked ({request.url.path})")
+
+ _orig_send = httpx.Client.send
+
+ def _send(self, request, *args, **kwargs):
+ _guard(request)
+ response = None
+ try:
+ response = _orig_send(self, request, *args, **kwargs)
+ return response
+ finally:
+ _log(request, response)
+
+ httpx.Client.send = _send
+
+ _orig_send_async = httpx.AsyncClient.send
+
+ async def _send_async(self, request, *args, **kwargs):
+ _guard(request)
+ response = None
+ try:
+ response = await _orig_send_async(self, request, *args, **kwargs)
+ return response
+ finally:
+ _log(request, response)
+
+ httpx.AsyncClient.send = _send_async
+ except Exception: # noqa: BLE001 - tracing must never break the example
+ pass
diff --git a/harness/trace/register.mjs b/harness/trace/register.mjs
new file mode 100644
index 000000000..7c0e6adc6
--- /dev/null
+++ b/harness/trace/register.mjs
@@ -0,0 +1,73 @@
+// Harness-owned fetch tracer. Injected via NODE_OPTIONS=--import; examples
+// must never reference COMPOSIO_TRACE_FILE themselves (lint-enforced).
+import { requireStagingBaseUrl } from '../staging-backend.mjs';
+
+const traceFile = process.env.COMPOSIO_TRACE_FILE;
+
+if (traceFile) {
+ const { appendFileSync } = await import('node:fs');
+
+ const LLM_HOSTS = new Set([
+ 'api.openai.com',
+ 'api.anthropic.com',
+ 'generativelanguage.googleapis.com',
+ ]);
+
+ const backendHost = new URL(requireStagingBaseUrl()).hostname;
+
+ const ID_SEG =
+ /^(ca_|ac_|ti_|tr_|trs_|sess_|auth_|req_|proj_|org_)[\w-]+$|^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$|^\d+$/i;
+ const template = (path) =>
+ path
+ .split('/')
+ .map((seg) => (ID_SEG.test(seg) ? '{id}' : seg))
+ .join('/');
+
+ const record = (obj) => {
+ try {
+ appendFileSync(traceFile, `${JSON.stringify(obj)}\n`);
+ } catch {
+ // never break the example over tracing
+ }
+ };
+
+ // Outbound-email guard: tool executions matching the denylist are refused at
+ // the transport, never forwarded to the backend.
+ const DENY = new RegExp(process.env.COMPOSIO_TOOL_DENYLIST ?? 'GMAIL_SEND|GMAIL_REPLY|SEND_EMAIL|SEND_DRAFT|OUTLOOK[A-Z_]*SEND', 'i');
+
+ const origFetch = globalThis.fetch;
+ globalThis.fetch = async function tracedFetch(input, init) {
+ let url;
+ try {
+ url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
+ } catch {
+ return origFetch.call(this, input, init);
+ }
+ const method = (
+ init?.method ??
+ (typeof input === 'object' && input !== null && 'method' in input ? input.method : 'GET') ??
+ 'GET'
+ ).toUpperCase();
+
+ if (url.hostname === backendHost && DENY.test(url.pathname)) {
+ record({ m: method, p: template(url.pathname), s: 'BLOCKED' });
+ throw new Error(`harness: outbound-email tool execution blocked (${url.pathname})`);
+ }
+
+ let res;
+ try {
+ res = await origFetch.call(this, input, init);
+ return res;
+ } finally {
+ if (url.hostname === backendHost) {
+ record({
+ m: method,
+ p: template(url.pathname),
+ s: res ? `${Math.floor(res.status / 100)}xx` : 'ERR',
+ });
+ } else if (LLM_HOSTS.has(url.hostname)) {
+ record({ llm: url.hostname });
+ }
+ }
+ };
+}
diff --git a/parity-variance.json b/parity-variance.json
new file mode 100644
index 000000000..046955d4b
--- /dev/null
+++ b/parity-variance.json
@@ -0,0 +1,3 @@
+{
+ "entries": []
+}
diff --git a/scripts/examples-provision.mjs b/scripts/examples-provision.mjs
index 275955164..f62773c0c 100644
--- a/scripts/examples-provision.mjs
+++ b/scripts/examples-provision.mjs
@@ -30,34 +30,23 @@
// values are ever printed. The API-key demo value stored for serpapi is a
// deliberately fake placeholder, not a real key.
-const BASE_URL = process.env.COMPOSIO_BASE_URL ?? 'https://backend.composio.dev';
+import { requireStagingBaseUrl } from '../harness/staging-backend.mjs';
+
+let BASE_URL;
+try {
+ BASE_URL = requireStagingBaseUrl();
+} catch (error) {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+}
const API_KEY = process.env.COMPOSIO_API_KEY;
const USER_ID = process.env.COMPOSIO_EXAMPLES_USER_ID ?? 'examples';
const INITIATE_MISSING = process.argv.includes('--initiate-missing');
const GC = process.argv.includes('--gc');
const DRY_RUN = process.argv.includes('--dry-run');
-// The examples only ever run against the production backend with the
-// disposable project key. Allowlist the Composio hosts rather than denylisting
-// loopback spellings: the API key travels in every request, so anything that is
-// not a Composio backend must be refused, not just localhost.
-const { baseHost, baseProtocol } = (() => {
- try {
- const url = new URL(BASE_URL);
- return { baseHost: url.hostname, baseProtocol: url.protocol };
- } catch {
- return { baseHost: null, baseProtocol: null };
- }
-})();
-if (!baseHost || !(baseHost === 'composio.dev' || baseHost.endsWith('.composio.dev'))) {
- console.error(`refusing non-Composio COMPOSIO_BASE_URL: ${BASE_URL}`);
- process.exit(1);
-}
-if (baseProtocol !== 'https:') {
- // The API key is sent on every request. Plain http would put it on the wire.
- console.error(`refusing non-https COMPOSIO_BASE_URL: ${BASE_URL}`);
- process.exit(1);
-}
+// The examples only ever run against staging with a disposable project key.
+// requireStagingBaseUrl refuses every other host before the key is read or sent.
if (!API_KEY) {
console.error('COMPOSIO_API_KEY is required (dedicated examples-project key)');
process.exit(1);