Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
feat(examples): live-sweep harness, manifest, and scheduled workflow (#4117)
Alberto Schiabel71ec523d258b
diff --git a/.github/workflows/examples-live.yml b/.github/workflows/examples-live.ymlnew file mode 100644index 000000000..f9e456cea--- /dev/null+++ b/.github/workflows/examples-live.yml@@ -0,0 +1,167 @@+name: Examples Manifest Sweep (staging)++# Runs the example entrypoints from examples-manifest.json against the+# STAGING backend using a dedicated, disposable examples project — never+# the org's main key. Non-blocking by design: scheduled + on demand, never a+# required PR check. Local runs and CI share the same code path+# (harness/run.mjs sweep).+on:+ schedule:+ - cron: '0 7 * * *' # 07:00 UTC nightly+ workflow_dispatch:+ inputs:+ lang:+ description: 'Language to sweep'+ type: choice+ options: [all, ts, py]+ default: all+ client:+ description: 'Client under test'+ type: choice+ options: [baseline, candidate]+ default: baseline+ ids:+ description: 'Comma-separated entry ids (empty = full manifest)'+ type: string+ default: ''+ client_tarball_url:+ description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)'+ type: string+ default: ''+ client_wheel_url:+ description: 'Candidate Python client wheel URL (required for Python candidate entries)'+ type: string+ default: ''+ llm:+ description: 'LLM traffic: mock (aimock, zero model spend) or live'+ type: choice+ options: [mock, live]+ default: mock++concurrency:+ group: ${{ github.workflow }}+ cancel-in-progress: false++permissions:+ contents: read++env:+ # Staging URL is not a secret; the keys are. The key must belong to a+ # project that holds no valuable connections or data — never the org's+ # main project. Example runs create their own resources and only ever+ # delete what they created.+ COMPOSIO_BASE_URL: https://staging-backend.composio.dev+ COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ # mock unless explicitly dispatched live; the nightly adds a small live canary.+ LLM_MODE: ${{ inputs.llm || 'mock' }}+ # Entries mocks cannot cover (llmMock:false) plus one representative+ # tool-calling example per language: the drift signal mocks can't see.+ CANARY_IDS: ts/openai/agents-api,ts/openai/agents-api-tool-router,ts/openai/chat-completions,py/tool_router/langchain_agent++jobs:+ examples-live:+ name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }})+ environment: staging+ runs-on: ubuntu-latest+ timeout-minutes: 60++ steps:+ - name: Checkout Code+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1++ - name: Setup Node.js, pnpm, Bun+ uses: ./.github/actions/setup-node-pnpm-bun+ with:+ enable-turbo-cache: 'true'++ - name: Setup Python and uv+ uses: ./.github/actions/setup-python-uv++ - name: Require credentials+ run: |+ if [ -z "$COMPOSIO_API_KEY" ]; then+ echo "::error::COMPOSIO_API_KEY is required."+ exit 1+ fi+ # LLM keys are only needed when live model traffic will run:+ # an explicit live dispatch, or the nightly's live canary step.+ if [ "$LLM_MODE" = "live" ] || [ "${{ github.event_name }}" = "schedule" ]; then+ if [ -z "$OPENAI_API_KEY" ] || [ -z "$ANTHROPIC_API_KEY" ]; then+ echo "::error::OPENAI_API_KEY and ANTHROPIC_API_KEY are required for live LLM runs."+ exit 1+ fi+ fi++ - name: Install Dependencies+ run: pnpm install --frozen-lockfile++ - name: Build SDK packages+ run: pnpm run build:packages++ - name: Fetch candidate client artifacts+ if: ${{ inputs.client == 'candidate' }}+ env:+ CLIENT_TARBALL_URL: ${{ inputs.client_tarball_url }}+ CLIENT_WHEEL_URL: ${{ inputs.client_wheel_url }}+ LANG_INPUT: ${{ inputs.lang || 'all' }}+ run: |+ set -euo pipefail+ mkdir -p .artifacts/candidate+ if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "ts" ]; then+ if [[ "$CLIENT_TARBALL_URL" != https://* ]]; then+ echo "::error::TS candidate entries require an HTTPS client_tarball_url"+ exit 1+ fi+ curl -fsSL -o .artifacts/candidate/composio-client.tgz "$CLIENT_TARBALL_URL"+ echo "COMPOSIO_CLIENT_TARBALL=$PWD/.artifacts/candidate/composio-client.tgz" >> "$GITHUB_ENV"+ fi+ if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "py" ]; then+ if [[ "$CLIENT_WHEEL_URL" != https://* ]]; then+ echo "::error::Python candidate entries require an HTTPS client_wheel_url"+ exit 1+ fi+ curl -fsSL -o .artifacts/candidate/composio-client.whl "$CLIENT_WHEEL_URL"+ echo "COMPOSIO_CLIENT_WHEEL=$PWD/.artifacts/candidate/composio-client.whl" >> "$GITHUB_ENV"+ fi++ - name: Harness selftest+ run: node harness/run.mjs selftest++ - name: Verify provisioned project state+ run: node scripts/examples-provision.mjs --gc > /tmp/examples-provision.env++ - name: Sweep+ env:+ CLIENT_INPUT: ${{ inputs.client || 'baseline' }}+ IDS_INPUT: ${{ inputs.ids }}+ LANG_INPUT: ${{ inputs.lang || 'all' }}+ run: |+ set -euo pipefail+ # Load the provisioned COMPOSIO_EXAMPLES_* ids emitted above.+ source /tmp/examples-provision.env+ ARGS=(--client "$CLIENT_INPUT" --llm "$LLM_MODE")+ if [ "$LANG_INPUT" != "all" ]; then+ ARGS+=(--lang "$LANG_INPUT")+ fi+ if [ -n "$IDS_INPUT" ]; then+ ARGS+=(--ids "$IDS_INPUT")+ fi+ node harness/run.mjs sweep "${ARGS[@]}"++ - name: Live LLM canary+ if: ${{ github.event_name == 'schedule' }}+ run: |+ set -euo pipefail+ source /tmp/examples-provision.env+ node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS"++ - name: Upload sweep results and traces+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: examples-live-results+ path: .artifacts/examples-parity/+ retention-days: 14+ if-no-files-found: ignorediff --git a/examples-manifest.json b/examples-manifest.jsonnew file mode 100644index 000000000..aae276a32--- /dev/null+++ b/examples-manifest.json@@ -0,0 +1,1294 @@+{+ "$comment": "Inventory of runnable example entrypoints, consumed by harness/run.mjs. Tiers: 1=unattended, 2=needs provisioned state (scripts/examples-provision.mjs), 3=bounded (pass at readiness match, then terminate), X=excluded. Readiness regexes must anchor on a labeled output line the example prints on success (e.g. 'Visit this URL to authorize: https?://'), never a bare 'https?://' — a URL inside an error message must not count as ready. llmMock:false marks entries whose LLM traffic cannot run against the mock server (live canary only).",+ "entries": [+ {+ "id": "ts/anthropic/index",+ "lang": "ts",+ "pkg": "anthropic-example",+ "file": "ts/examples/anthropic/src/index.ts",+ "tier": "1",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/anthropic/streaming",+ "lang": "ts",+ "pkg": "anthropic-example",+ "file": "ts/examples/anthropic/src/streaming.ts",+ "tier": "1",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/anthropic/mcp",+ "lang": "ts",+ "pkg": "anthropic-example",+ "file": "ts/examples/anthropic/src/mcp.ts",+ "tier": "2",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/anthropic/experimental.mcp",+ "lang": "ts",+ "pkg": "anthropic-example",+ "file": "ts/examples/anthropic/src/experimental.mcp.ts",+ "tier": "2",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/anthropic/claude-agents-sdk",+ "lang": "ts",+ "pkg": "anthropic-example",+ "file": "ts/examples/anthropic/src/claude-agents-sdk.ts",+ "tier": "2",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 300,+ "notes": "needs @anthropic-ai/claude-code CLI on PATH"+ },+ {+ "id": "ts/cloudflare-wrangler/dev",+ "lang": "ts",+ "pkg": "cloudflare-wrangler-example",+ "file": "ts/examples/cloudflare-wrangler/src/index.ts",+ "tier": "3",+ "script": "dev",+ "readiness": "Ready on|Listening",+ "toolkits": [+ "googledrive"+ ],+ "timeoutSec": 120,+ "notes": "wrangler dev; env must reach .dev.vars"+ },+ {+ "id": "ts/connected-accounts/index",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/index.ts",+ "tier": "3",+ "readiness": "authorize the user: https?://",+ "toolkits": [+ "github"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/connected-accounts/api-key",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/api-key.ts",+ "tier": "2",+ "ids": [+ "COMPOSIO_EXAMPLES_APIKEY_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_APIKEY_PLACEHOLDER",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [],+ "timeoutSec": 90+ },+ {+ "id": "ts/connected-accounts/magic-flow-demo",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/magic-flow-demo.ts",+ "tier": "3",+ "env": [+ "OPENAI_API_KEY"+ ],+ "readiness": "Redirect the user to https?://",+ "timeoutSec": 120+ },+ {+ "id": "ts/connected-accounts/multiple-connected-accounts",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/multiple-connected-accounts.ts",+ "tier": "2",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/connected-accounts/scopes",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/scopes.ts",+ "tier": "2",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/connected-accounts/toolkit-authorize",+ "lang": "ts",+ "pkg": "connected-accounts-example",+ "file": "ts/examples/connected-accounts/src/toolkit-authorize.ts",+ "tier": "3",+ "readiness": "authorize the toolkit: https?://",+ "toolkits": [+ "github"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/error-handling-demo/index",+ "lang": "ts",+ "pkg": "error-handling-example",+ "file": "ts/examples/error-handling-demo/src/index.ts",+ "tier": "1",+ "backend": false,+ "timeoutSec": 60+ },+ {+ "id": "ts/file-handling/index",+ "lang": "ts",+ "pkg": "file-handling-example",+ "file": "ts/examples/file-handling/src/index.ts",+ "tier": "X",+ "toolkits": [+ "gmail"+ ],+ "notes": "sends real email by design; excluded until a non-send variant is decided by the human",+ "reason": "outbound-email"+ },+ {+ "id": "ts/file-handling/url",+ "lang": "ts",+ "pkg": "file-handling-example",+ "file": "ts/examples/file-handling/src/url.ts",+ "tier": "X",+ "toolkits": [+ "gmail"+ ],+ "reason": "outbound-email",+ "notes": "sends real email by design; excluded until a non-send variant is decided by the human"+ },+ {+ "id": "ts/google/index",+ "lang": "ts",+ "pkg": "google-example",+ "file": "ts/examples/google/src/index.ts",+ "tier": "X",+ "env": [+ "GEMINI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "reason": "dropped-provider",+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"+ },+ {+ "id": "ts/google/experimental.mcp",+ "lang": "ts",+ "pkg": "google-example",+ "file": "ts/examples/google/src/experimental.mcp.ts",+ "tier": "X",+ "env": [+ "GEMINI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "reason": "dropped-provider",+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"+ },+ {+ "id": "ts/json-schema-to-zod/index",+ "lang": "ts",+ "pkg": "json-schema-to-zod-example",+ "file": "ts/examples/json-schema-to-zod/src/index.ts",+ "tier": "1",+ "timeoutSec": 90+ },+ {+ "id": "ts/langchain/index",+ "lang": "ts",+ "pkg": "langchain-example",+ "file": "ts/examples/langchain/src/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/langchain/openai",+ "lang": "ts",+ "pkg": "langchain-example",+ "file": "ts/examples/langchain/src/openai.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 300+ },+ {+ "id": "ts/llamaindex/index",+ "lang": "ts",+ "pkg": "llamaindex-example",+ "file": "ts/examples/llamaindex/src/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/mastra/index",+ "lang": "ts",+ "pkg": "mastra-example",+ "file": "ts/examples/mastra/src/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/mastra/tool-router",+ "lang": "ts",+ "pkg": "mastra-example",+ "file": "ts/examples/mastra/src/tool-router.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/mastra/smoke",+ "lang": "ts",+ "pkg": "mastra-example",+ "file": "ts/examples/mastra/src/smoke.ts",+ "tier": "1",+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/mastra/cloudflare",+ "lang": "ts",+ "pkg": "mastra-example",+ "file": "ts/examples/mastra/src/cloudflare.ts",+ "tier": "X",+ "reason": "worker-module",+ "notes": "covered by cf:dry-run in ts.examples.yml"+ },+ {+ "id": "ts/mcp/index",+ "lang": "ts",+ "pkg": "mcp-example",+ "file": "ts/examples/mcp/src/index.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/modifiers/index",+ "lang": "ts",+ "pkg": "modifiers-example",+ "file": "ts/examples/modifiers/src/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/index",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/tool-router",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/tool-router.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/smoke",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/smoke.ts",+ "tier": "1",+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/openai/chat-completions",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/chat-completions.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/tools",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/tools.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "googledrive"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/cloudflare",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/cloudflare.ts",+ "tier": "X",+ "reason": "worker-module",+ "notes": "covered by cf:dry-run in ts.examples.yml"+ },+ {+ "id": "ts/openai/agents-api",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/agents-api/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180,+ "llmMock": false,+ "notes": "@openai/agents rejects aimock responses-api output; live canary only."+ },+ {+ "id": "ts/openai/agents-api-mcp",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/agents-api/experiment.mcp.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/agents-api-tool-router",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/agents-api/experiment.tool-router.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180,+ "llmMock": false,+ "notes": "@openai/agents rejects aimock responses-api output; live canary only."+ },+ {+ "id": "ts/openai/responses-api",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/responses-api/index.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/openai/responses-api-mcp",+ "lang": "ts",+ "pkg": "openai-example",+ "file": "ts/examples/openai/src/responses-api/experiment.mcp.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/session-management/index",+ "lang": "ts",+ "pkg": "session-management-example",+ "file": "ts/examples/session-management/src/index.ts",+ "tier": "1",+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/tool-router/index",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/index.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/tool-router/authorize",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/authorize.ts",+ "tier": "3",+ "readiness": "Visit this URL to authorize: https?://",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 90+ },+ {+ "id": "ts/tool-router/claude-agent-sdk",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/claude-agent-sdk.ts",+ "tier": "2",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 300,+ "notes": "needs @anthropic-ai/claude-code CLI on PATH"+ },+ {+ "id": "ts/tool-router/custom-tools",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/custom-tools.ts",+ "tier": "2",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/tool-router/custom-tools-agent",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/custom-tools-agent.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/tool-router/custom-tools-e2e",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/custom-tools-e2e.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "weathermap",+ "gmail"+ ],+ "timeoutSec": 240+ },+ {+ "id": "ts/tool-router/direct-tools-preset",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/direct-tools-preset.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/tool-router/langchain",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/langchain.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "ts/tool-router/mcp",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/mcp.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/tool-router/multi-account",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/multi-account.ts",+ "tier": "3",+ "env": [+ "OPENAI_API_KEY"+ ],+ "readiness": "Visit this URL to authorize: https?://",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 120,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "ts/tool-router/openai-agents",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/openai-agents.ts",+ "tier": "X",+ "reason": "stdin-repl"+ },+ {+ "id": "ts/tool-router/preload",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/preload.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/tool-router/session-update",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/session-update.ts",+ "tier": "1",+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/tool-router/toolkits",+ "lang": "ts",+ "pkg": "tool-router-example",+ "file": "ts/examples/tool-router/src/toolkits.ts",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "ts/toolkits/index",+ "lang": "ts",+ "pkg": "toolkits-example",+ "file": "ts/examples/toolkits/src/index.ts",+ "tier": "3",+ "env": [+ "OPENAI_API_KEY"+ ],+ "readiness": "authorize the toolkit: https?://",+ "toolkits": [+ "github"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/toolkits/toolkits",+ "lang": "ts",+ "pkg": "toolkits-example",+ "file": "ts/examples/toolkits/src/toolkits.ts",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "ts/toolkits/auth-fields",+ "lang": "ts",+ "pkg": "toolkits-example",+ "file": "ts/examples/toolkits/src/auth-fields.ts",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "ts/tools/index",+ "lang": "ts",+ "pkg": "tools-example",+ "file": "ts/examples/tools/src/index.ts",+ "tier": "2",+ "toolkits": [+ "googledrive"+ ],+ "timeoutSec": 180,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "ts/tools/ai",+ "lang": "ts",+ "pkg": "tools-example",+ "file": "ts/examples/tools/src/ai.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "googledrive"+ ],+ "timeoutSec": 180,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "ts/tools/tool-conversion",+ "lang": "ts",+ "pkg": "tools-example",+ "file": "ts/examples/tools/src/tool-conversion.ts",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "ts/triggers/index",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/index.ts",+ "tier": "3",+ "readiness": "Subscribed to trigger events",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/triggers/create",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/create.ts",+ "tier": "2",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "serial": true,+ "timeoutSec": 120+ },+ {+ "id": "ts/triggers/subscribe",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/subscribe.ts",+ "tier": "3",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "readiness": "Subscribed to triggers",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 120+ },+ {+ "id": "ts/triggers/test",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/test.ts",+ "tier": "2",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 120,+ "notes": "triage: entry not yet characterized"+ },+ {+ "id": "ts/triggers/trigger-types",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/trigger-types.ts",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "ts/triggers/webhook-server",+ "lang": "ts",+ "pkg": "triggers-example",+ "file": "ts/examples/triggers/src/webhook-server.ts",+ "tier": "X",+ "reason": "external-tunnel"+ },+ {+ "id": "ts/vercel/ai",+ "lang": "ts",+ "pkg": "vercel-example",+ "file": "ts/examples/vercel/src/ai.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/vercel/stream",+ "lang": "ts",+ "pkg": "vercel-example",+ "file": "ts/examples/vercel/src/stream.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/vercel/tool-loop-agent",+ "lang": "ts",+ "pkg": "vercel-example",+ "file": "ts/examples/vercel/src/tool-loop-agent.ts",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 240+ },+ {+ "id": "ts/vercel/tool-router-ai",+ "lang": "ts",+ "pkg": "vercel-example",+ "file": "ts/examples/vercel/src/tool-router-ai.ts",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180+ },+ {+ "id": "ts/versioning/index",+ "lang": "ts",+ "pkg": "versioning-example",+ "file": "ts/examples/versioning/src/index.ts",+ "tier": "1",+ "toolkits": [+ "gmail",+ "googlecalendar"+ ],+ "timeoutSec": 120+ },+ {+ "id": "py/tools",+ "lang": "py",+ "file": "python/examples/tools.py",+ "tier": "2",+ "ids": [+ "COMPOSIO_EXAMPLES_GITHUB_CONNECTED_ACCOUNT_ID"+ ],+ "toolkits": [+ "github"+ ],+ "timeoutSec": 120+ },+ {+ "id": "py/toolkits",+ "lang": "py",+ "file": "python/examples/toolkits.py",+ "tier": "2",+ "toolkits": [+ "github"+ ],+ "serial": true,+ "timeoutSec": 120,+ "notes": "authorize() initiates OAuth state"+ },+ {+ "id": "py/auth_configs",+ "lang": "py",+ "file": "python/examples/auth_configs.py",+ "tier": "2",+ "toolkits": [],+ "serial": true,+ "timeoutSec": 120,+ "notes": "creates/updates/deletes org auth configs"+ },+ {+ "id": "py/connected_accounts",+ "lang": "py",+ "file": "python/examples/connected_accounts.py",+ "tier": "3",+ "readiness": "Visit this URL to authorize: https?://",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_APIKEY_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_APIKEY_PLACEHOLDER",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 90+ },+ {+ "id": "py/triggers",+ "lang": "py",+ "file": "python/examples/triggers.py",+ "tier": "3",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_CONNECTED_ACCOUNT_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "readiness": "Subscribed|[Ll]istening for events",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 300+ },+ {+ "id": "py/modifiers",+ "lang": "py",+ "file": "python/examples/modifiers.py",+ "tier": "X",+ "env": [+ "NOTION_API_KEY"+ ],+ "toolkits": [+ "hackernews",+ "notion"+ ],+ "reason": "dropped-provider",+ "notes": "gemini/notion are out of scope for this goal by decision (2026-08-10)"+ },+ {+ "id": "py/mcp",+ "lang": "py",+ "file": "python/examples/mcp_example.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "ids": [+ "COMPOSIO_EXAMPLES_SLACK_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "toolkits": [+ "slack"+ ],+ "timeoutSec": 180,+ "pyWith": [+ "langchain-mcp-adapters",+ "langgraph",+ "langchain",+ "langchain-openai"+ ]+ },+ {+ "id": "py/fastapi_app",+ "lang": "py",+ "file": "python/examples/fastapi_app.py",+ "tier": "3",+ "readiness": "Uvicorn running|Application startup",+ "ids": [+ "COMPOSIO_EXAMPLES_GMAIL_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_USER_ID"+ ],+ "pyWith": [+ "fastapi",+ "uvicorn"+ ],+ "timeoutSec": 120+ },+ {+ "id": "py/custom_tools_agent_test",+ "lang": "py",+ "file": "python/examples/custom_tools_agent_test.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 240,+ "pyWith": [+ "openai-agents>=0.19"+ ]+ },+ {+ "id": "py/experimental_tool_router",+ "lang": "py",+ "file": "python/examples/experimental_tool_router_example.py",+ "tier": "3",+ "readiness": "Redirect URL: https?://",+ "toolkits": [+ "github"+ ],+ "timeoutSec": 120+ },+ {+ "id": "py/experimental_tool_router_advanced",+ "lang": "py",+ "file": "python/examples/experimental_tool_router_advanced_example.py",+ "tier": "1",+ "toolkits": [+ "github",+ "slack",+ "linear",+ "jira"+ ],+ "timeoutSec": 120,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID",+ "COMPOSIO_EXAMPLES_GITHUB_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_SLACK_AUTH_CONFIG_ID",+ "COMPOSIO_EXAMPLES_GITHUB_CONNECTED_ACCOUNT_ID",+ "COMPOSIO_EXAMPLES_SLACK_CONNECTED_ACCOUNT_ID"+ ]+ },+ {+ "id": "py/tool_router/authorize",+ "lang": "py",+ "file": "python/examples/tool_router/authorize.py",+ "tier": "3",+ "readiness": "Redirect URL: https?://",+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 90+ },+ {+ "id": "py/tool_router/claude_agent",+ "lang": "py",+ "file": "python/examples/tool_router/claude_agent.py",+ "tier": "2",+ "env": [+ "ANTHROPIC_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "pyWith": [+ "claude-agent-sdk",+ "composio-claude-agent-sdk",+ "./python"+ ],+ "timeoutSec": 300,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "py/tool_router/crewai_agent",+ "lang": "py",+ "file": "python/examples/tool_router/crewai_agent.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "pyWith": [+ "crewai>=1.15"+ ],+ "timeoutSec": 300,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "py/tool_router/direct_tools_preset",+ "lang": "py",+ "file": "python/examples/tool_router/direct_tools_preset.py",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180,+ "pyWith": [+ "openai-agents>=0.19"+ ]+ },+ {+ "id": "py/tool_router/preload",+ "lang": "py",+ "file": "python/examples/tool_router/preload.py",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "hackernews"+ ],+ "timeoutSec": 180,+ "pyWith": [+ "openai-agents>=0.19"+ ]+ },+ {+ "id": "py/tool_router/files",+ "lang": "py",+ "file": "python/examples/tool_router/files.py",+ "tier": "1",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [],+ "timeoutSec": 180+ },+ {+ "id": "py/tool_router/langchain_agent",+ "lang": "py",+ "file": "python/examples/tool_router/langchain_agent.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "pyWith": [+ "langchain",+ "langchain-openai",+ "langchain-mcp-adapters"+ ],+ "timeoutSec": 240,+ "ids": [+ "COMPOSIO_EXAMPLES_USER_ID"+ ]+ },+ {+ "id": "py/tool_router/openai_agents",+ "lang": "py",+ "file": "python/examples/tool_router/openai_agents.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "pyWith": [+ "openai-agents>=0.19"+ ],+ "timeoutSec": 240+ },+ {+ "id": "py/tool_router/tool_router_mcp",+ "lang": "py",+ "file": "python/examples/tool_router/tool_router_mcp.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "github",+ "gmail"+ ],+ "timeoutSec": 240,+ "pyWith": [+ "openai-agents>=0.19"+ ]+ },+ {+ "id": "py/tool_router/session_update",+ "lang": "py",+ "file": "python/examples/tool_router/session_update.py",+ "tier": "1",+ "toolkits": [+ "gmail",+ "github"+ ],+ "timeoutSec": 120+ },+ {+ "id": "py/tool_router/toolkits",+ "lang": "py",+ "file": "python/examples/tool_router/toolkits.py",+ "tier": "1",+ "toolkits": [],+ "timeoutSec": 120+ },+ {+ "id": "py/tool_router/tools",+ "lang": "py",+ "file": "python/examples/tool_router/tools.py",+ "tier": "2",+ "env": [+ "OPENAI_API_KEY"+ ],+ "toolkits": [+ "gmail"+ ],+ "timeoutSec": 180,+ "pyWith": [+ "openai-agents>=0.19"+ ]+ }+ ]+}diff --git a/harness/fixtures/always-fails.ts b/harness/fixtures/always-fails.tsnew file mode 100644index 000000000..9e49d7085--- /dev/null+++ b/harness/fixtures/always-fails.ts@@ -0,0 +1,3 @@+// Known-bad selftest fixture: must exit non-zero.+console.error('fixture: failing on purpose');+process.exit(1);diff --git a/harness/fixtures/always_fails.py b/harness/fixtures/always_fails.pynew file mode 100644index 000000000..21b115275--- /dev/null+++ b/harness/fixtures/always_fails.py@@ -0,0 +1,6 @@+"""Known-bad selftest fixture: must exit non-zero."""++import sys++print("fixture: failing on purpose", file=sys.stderr)+sys.exit(1)diff --git a/harness/fixtures/trace-check.ts b/harness/fixtures/trace-check.tsnew file mode 100644index 000000000..c6a47369b--- /dev/null+++ b/harness/fixtures/trace-check.ts@@ -0,0 +1,7 @@+// Selftest fixture: makes one unauthenticated request against the backend so+// the fetch shim must record a non-2xx composio line in COMPOSIO_TRACE_FILE.+const base = process.env.COMPOSIO_BASE_URL ?? 'https://staging-backend.composio.dev';+const res = await fetch(`${base}/api/v3/toolkits`, {+ headers: { 'x-api-key': 'selftest-invalid-key' },+});+console.log(`trace-check: status ${res.status}`);diff --git a/harness/fixtures/trace_check.py b/harness/fixtures/trace_check.pynew file mode 100644index 000000000..c29a4f64a--- /dev/null+++ b/harness/fixtures/trace_check.py@@ -0,0 +1,11 @@+"""Selftest fixture: one unauthenticated httpx request against the backend so+the sitecustomize shim must record a non-2xx composio line."""++import os++import httpx++base = os.environ.get("COMPOSIO_BASE_URL", "https://staging-backend.composio.dev")+with httpx.Client() as client:+ response = client.get(f"{base}/api/v3/toolkits", headers={"x-api-key": "selftest-invalid-key"})+print(f"trace-check: status {response.status_code}")diff --git a/harness/llm-mock/aimock.json b/harness/llm-mock/aimock.jsonnew file mode 100644index 000000000..fc516d466--- /dev/null+++ b/harness/llm-mock/aimock.json@@ -0,0 +1,5 @@+{+ "llm": {+ "fixtures": "./fixtures"+ }+}diff --git a/harness/llm-mock/fixtures/default.json b/harness/llm-mock/fixtures/default.jsonnew file mode 100644index 000000000..7e6e813c4--- /dev/null+++ b/harness/llm-mock/fixtures/default.json@@ -0,0 +1,29 @@+{+ "$comment": "Deterministic LLM fixtures for --llm mock sweeps. One file so fixture order is explicit: specific matches first, catch-alls last (first match wins). Tool-call fixtures guard with hasToolResult:false so the follow-up turn falls through to the wrap-up fixture. Composio tool execution stays live; only the model is mocked.",+ "fixtures": [+ {+ "match": { "userMessage": "HackerNews user", "hasToolResult": false },+ "response": {+ "toolCalls": [+ { "name": "HACKERNEWS_GET_USER", "arguments": { "username": "pg" }, "id": "call_hn_user" }+ ]+ }+ },+ {+ "match": { "userMessage": "HackerNews", "hasToolResult": false },+ "response": {+ "toolCalls": [+ { "name": "HACKERNEWS_GET_TOP_STORIES", "arguments": { "size": 3 }, "id": "call_hn_top" }+ ]+ }+ },+ {+ "match": { "hasToolResult": true },+ "response": { "content": "Mock model wrap-up: summarized the tool result deterministically." }+ },+ {+ "match": {},+ "response": { "content": "Mock model response: no tool needed for this prompt." }+ }+ ]+}diff --git a/harness/parity.mjs b/harness/parity.mjsnew file mode 100755index 000000000..89b73dda8--- /dev/null+++ b/harness/parity.mjs@@ -0,0 +1,73 @@+#!/usr/bin/env node+// Trace-parity comparator: node harness/parity.mjs <baseline-run-dir> <candidate-run-dir>+// Per entry green in BOTH runs: the sets of distinct (method, path-template)+// pairs must be equal, ignoring pairs allowlisted in parity-variance.json.+// Prints a JSON report; exit 0 = all parities hold, 1 = mismatches.++import { readFileSync, readdirSync, existsSync } from 'node:fs';+import { join, resolve, dirname } from 'node:path';+import { fileURLToPath } from 'node:url';++const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');+const [baseDir, candDir] = process.argv.slice(2);+if (!baseDir || !candDir) {+ console.error('usage: parity.mjs <baseline-run-dir> <candidate-run-dir>');+ process.exit(2);+}++const variance = JSON.parse(readFileSync(join(ROOT, 'parity-variance.json'), 'utf8'));+const ignored = new Set((variance.entries ?? []).map((e) => e.pair));++const loadRun = (dir) => {+ const results = readFileSync(join(dir, 'results.jsonl'), 'utf8')+ .trim()+ .split('\n')+ .filter(Boolean)+ .map((l) => JSON.parse(l));+ const byId = new Map();+ for (const r of results) byId.set(r.id, r);+ return byId;+};++const tracePairs = (dir, id) => {+ const file = join(dir, 'traces', `${id.replace(/[/]/g, '__')}.jsonl`);+ const pairs = new Set();+ if (!existsSync(file)) return pairs;+ for (const line of readFileSync(file, 'utf8').trim().split('\n').filter(Boolean)) {+ const rec = JSON.parse(line);+ if (rec.m && rec.p) pairs.add(`${rec.m} ${rec.p}`);+ }+ return pairs;+};++const baseline = loadRun(baseDir);+const candidate = loadRun(candDir);++const report = [];+const ids = new Set([...baseline.keys(), ...candidate.keys()]);+for (const id of [...ids].sort()) {+ const b = baseline.get(id);+ const c = candidate.get(id);+ if (!b || !c) {+ report.push({+ id,+ parity: false,+ reason: `missing ${b ? 'candidate' : 'baseline'} result`,+ });+ continue;+ }+ if (b.status !== 'green' || c.status !== 'green') {+ report.push({ id, parity: false, reason: `status baseline=${b.status} candidate=${c.status}` });+ continue;+ }+ const bp = tracePairs(baseDir, id);+ const cp = tracePairs(candDir, id);+ const onlyBaseline = [...bp].filter((p) => !cp.has(p) && !ignored.has(p));+ const onlyCandidate = [...cp].filter((p) => !bp.has(p) && !ignored.has(p));+ const match = onlyBaseline.length === 0 && onlyCandidate.length === 0;+ report.push({ id, parity: match, ...(match ? {} : { onlyBaseline, onlyCandidate }) });+}++const green = report.filter((r) => r.parity).length;+console.log(JSON.stringify({ compared: report.length, parityGreen: green, ignoredPairs: ignored.size, entries: report }, null, 2));+process.exit(report.length > 0 && report.every((r) => r.parity) ? 0 : 1);diff --git a/harness/run.mjs b/harness/run.mjsnew file mode 100755index 000000000..bf159fa6a--- /dev/null+++ b/harness/run.mjs@@ -0,0 +1,545 @@+#!/usr/bin/env node+// Examples runner: sweeps the entrypoints in examples-manifest.json against+// the live backend, recording per-entry results and Composio traces.+//+// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--llm live|mock]+// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--sample N] [--seed S]+// node harness/run.mjs selftest+//+// Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.++import { spawn } from 'node:child_process';+import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';+import { dirname, join, relative, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { requireStagingBaseUrl, STAGING_BASE_URL } from './staging-backend.mjs';++const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');+const ARTIFACTS = join(ROOT, '.artifacts', 'examples-parity');+const STAINLESS_TS_VERSION = '0.1.0-alpha.76';+const AIMOCK_VERSION = '1.38.0';+const AIMOCK_PORT = Number(process.env.AIMOCK_PORT ?? 4010);++// Set by cmdSweep when --llm mock: LLM traffic goes to a local aimock server+// and provider keys are overridden so a mock sweep can never spend tokens.+let LLM_MOCK = false;++const args = process.argv.slice(2);+const cmd = args[0];+const opt = (name, fallback) => {+ const i = args.indexOf(`--${name}`);+ return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : fallback;+};++class HarnessError extends Error {+ constructor(message, exitCode = 2) {+ super(message);+ this.exitCode = exitCode;+ }+}++const fail = (msg, code = 2) => {+ throw new HarnessError(msg, code);+};++const loadManifest = () => {+ const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));+ for (const e of manifest.entries) {+ if (!e.id || !e.lang || !e.file || !e.tier) fail(`manifest entry missing required fields: ${JSON.stringify(e)}`);+ if (e.lang === 'ts' && e.tier !== 'X' && !e.pkg) fail(`ts entry ${e.id} missing pkg`);+ if (e.tier === '3' && !e.readiness) fail(`tier-3 entry ${e.id} missing readiness regex`);+ }+ return manifest.entries;+};++const selectEntries = () => {+ const lang = opt('lang');+ const ids = opt('ids') ? opt('ids').split(',') : null;+ const tiers = (opt('tiers', '1,2,3')).split(',');+ return loadManifest().filter(+ (e) =>+ e.tier !== 'X' &&+ tiers.includes(e.tier) &&+ (!lang || e.lang === lang) &&+ (!ids || ids.includes(e.id))+ );+};++const baseUrl = () => {+ try {+ return requireStagingBaseUrl();+ } catch (error) {+ fail(error instanceof Error ? error.message : String(error));+ }+};++// Deterministic PRNG so --seed reproduces a negative-control sample.+const mulberry32 = (a) => () => {+ a |= 0; a = (a + 0x6d2b79f5) | 0;+ let t = Math.imul(a ^ (a >>> 15), 1 | a);+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;+};++const runId = () => new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) + '-' + Math.random().toString(36).slice(2, 8);++/** Spawn one entry; resolve {exit, readinessMatched, timedOut, durationMs}. */+const runEntry = (entry, env, { garbage = false } = {}) =>+ new Promise((resolveRun) => {+ const isTs = entry.lang === 'ts';+ let cmdline;+ if (isTs && entry.script) {+ cmdline = ['pnpm', '--filter', entry.pkg, 'run', entry.script];+ } else if (isTs) {+ // pnpm --filter runs with the example package as cwd; derive the package+ // dir from the entry file path (ts/examples/<dir>/src/...).+ const pkgDir = entry.file.split('/').slice(0, 3).join('/');+ const srcRel = relative(join(ROOT, pkgDir), join(ROOT, entry.file));+ cmdline = ['pnpm', '--filter', entry.pkg, 'exec', 'tsx', srcRel];+ } else {+ cmdline = ['uv', 'run', '--project', 'python'];+ for (const dep of entry.pyWith ?? []) cmdline.push('--with', dep);+ cmdline.push('python', join(ROOT, entry.file));+ }++ const child = spawn(cmdline[0], cmdline.slice(1), {+ cwd: ROOT,+ env,+ detached: true,+ stdio: ['ignore', 'pipe', 'pipe'],+ });++ const started = Date.now();+ const timeoutMs = (entry.timeoutSec ?? 120) * 1000;+ let readinessMatched = false;+ let timedOut = false;+ let settled = false;+ let outputTail = '';+ const readiness = entry.tier === '3' && entry.readiness ? new RegExp(entry.readiness) : null;++ const killTree = () => {+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }+ setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 10_000).unref();+ };++ const onChunk = (chunk) => {+ const text = chunk.toString();+ outputTail = (outputTail + text).slice(-4000);+ if (readiness && !readinessMatched && readiness.test(outputTail)) {+ readinessMatched = true;+ // Grace period so the in-flight backend call lands in the trace, then stop the listener.+ setTimeout(killTree, 3000).unref();+ }+ };+ child.stdout.on('data', onChunk);+ child.stderr.on('data', onChunk);++ const timer = setTimeout(() => {+ timedOut = true;+ killTree();+ }, timeoutMs);++ child.on('exit', (code, signal) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolveRun({+ exit: code ?? (signal ? 128 : 1),+ readinessMatched,+ timedOut,+ durationMs: Date.now() - started,+ outputTail: garbage ? undefined : outputTail.slice(-1500),+ });+ });+ child.on('error', () => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ resolveRun({ exit: 127, readinessMatched: false, timedOut: false, durationMs: Date.now() - started });+ });+ });++const isGreen = (entry, r) => {+ if (entry.tier === '3') return r.readinessMatched || (!r.timedOut && r.exit === 0);+ return !r.timedOut && r.exit === 0;+};++const buildEnv = (entry, runDir, { garbage = false } = {}) => {+ const traceFile = join(runDir, 'traces', `${entry.id.replace(/[/]/g, '__')}.jsonl`);+ mkdirSync(dirname(traceFile), { recursive: true });+ const env = { ...process.env };+ env.COMPOSIO_BASE_URL = baseUrl();+ env.COMPOSIO_TRACE_FILE = traceFile;+ env.COMPOSIO_LOG_LEVEL ??= 'error';+ // The outbound-email denylist is fixed by the shims; never overridable per run.+ delete env.COMPOSIO_TOOL_DENYLIST;+ if (entry.lang === 'ts') {+ env.NODE_OPTIONS = `${env.NODE_OPTIONS ?? ''} --import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}`.trim();+ } else {+ env.PYTHONPATH = `${join(ROOT, 'harness', 'trace-py')}${env.PYTHONPATH ? `:${env.PYTHONPATH}` : ''}`;+ }+ if (garbage) {+ env.COMPOSIO_API_KEY = 'nc-invalid-0000000000000000';+ env.OPENAI_API_KEY = 'nc-invalid';+ env.ANTHROPIC_API_KEY = 'nc-invalid';+ env.GEMINI_API_KEY = 'nc-invalid';+ env.NOTION_API_KEY = 'nc-invalid';+ } else if (LLM_MOCK) {+ const mock = `http://127.0.0.1:${AIMOCK_PORT}`;+ env.OPENAI_BASE_URL = `${mock}/v1`;+ // langchain/litellm read the older variable name.+ env.OPENAI_API_BASE = `${mock}/v1`;+ env.ANTHROPIC_BASE_URL = mock;+ env.OPENAI_API_KEY = 'aimock';+ env.ANTHROPIC_API_KEY = 'aimock';+ }+ return { env, traceFile };+};++const LLM_KEYS = new Set(['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GEMINI_API_KEY']);++const missingEnv = (entry) => {+ const wanted = [...(entry.env ?? []), ...(entry.ids ?? [])].filter(+ (k) => !(LLM_MOCK && LLM_KEYS.has(k))+ );+ if (entry.backend !== false) wanted.unshift('COMPOSIO_API_KEY');+ return wanted.filter((k) => !process.env[k]);+};++/** Run entries with bounded concurrency; serial entries run last, alone. */+const runPool = async (entries, runDir, { garbage = false, concurrency = 4 } = {}) => {+ const parallel = entries.filter((e) => !e.serial);+ const serial = entries.filter((e) => e.serial);+ const results = [];+ const worker = async (queue) => {+ for (;;) {+ const entry = queue.shift();+ if (!entry) return;+ if (LLM_MOCK && entry.llmMock === false) {+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: 'llm-mock-unsupported' });+ console.log(` ~ ${entry.id} skipped (llm-mock-unsupported)`);+ continue;+ }+ const missing = garbage ? [] : missingEnv(entry);+ if (missing.length > 0) {+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: `missing-env:${missing.join(',')}` });+ console.log(` ~ ${entry.id} skipped (${missing.join(',')})`);+ continue;+ }+ const { env, traceFile } = buildEnv(entry, runDir, { garbage });+ const r = await runEntry(entry, env, { garbage });+ // An entry that tripped the outbound-email guard is red no matter how it+ // exited: catching the refusal and continuing is not coverage.+ const blocked =+ existsSync(traceFile) && readFileSync(traceFile, 'utf8').includes('"s":"BLOCKED"');+ const green = !blocked && isGreen(entry, r);+ results.push({+ id: entry.id,+ tier: entry.tier,+ status: green ? 'green' : 'red',+ ...(blocked ? { reason: 'blocked-tool' } : {}),+ exit: r.exit,+ timedOut: r.timedOut,+ readinessMatched: r.readinessMatched,+ durationMs: r.durationMs,+ traceFile: relative(ROOT, traceFile),+ outputTail: green ? undefined : r.outputTail,+ });+ console.log(` ${green ? '✓' : '✗'} ${entry.id} (${(r.durationMs / 1000).toFixed(1)}s${r.timedOut ? ', timeout' : ''})`);+ }+ };+ const queue = [...parallel];+ await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(queue.length, 1)) }, () => worker(queue)));+ for (const entry of serial) await worker([entry]);+ return results;+};++// --- candidate swap (TS) --------------------------------------------------++const WORKSPACE_YAML = join(ROOT, 'pnpm-workspace.yaml');+const LOCKFILE = join(ROOT, 'pnpm-lock.yaml');++const sh = (cmdline, opts = {}) =>+ new Promise((resolveSh, rejectSh) => {+ const child = spawn(cmdline[0], cmdline.slice(1), { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'], ...opts });+ let out = '';+ child.stdout.on('data', (c) => (out += c));+ child.stderr.on('data', (c) => (out += c));+ child.on('exit', (code) => (code === 0 ? resolveSh(out) : rejectSh(new Error(`${cmdline.join(' ')} → exit ${code}\n${out.slice(-2000)}`))));+ });++const resolvedTsClientVersion = async () => {+ const out = await sh(['node', '-e', "console.log(JSON.parse(require('fs').readFileSync(require.resolve('@composio/client/package.json', {paths:[require('path').join(process.cwd(),'ts/packages/core')]}), 'utf8')).version)"]);+ return out.trim().split('\n').pop();+};++const applyTsCandidate = async (tarball) => {+ if (!existsSync(tarball)) fail(`COMPOSIO_CLIENT_TARBALL not found: ${tarball}`);+ const yaml = readFileSync(WORKSPACE_YAML, 'utf8');+ if (/^overrides:$/m.test(yaml)) {+ // Merge into the repo's existing overrides block (e.g. security pins);+ // restoreTsBaseline() writes back the exact pre-run snapshot either way.+ writeFileSync(WORKSPACE_YAML, yaml.replace(/^overrides:$/m, `overrides:\n '@composio/client': file:${tarball}`));+ } else {+ writeFileSync(WORKSPACE_YAML, `${yaml}\noverrides:\n '@composio/client': file:${tarball}\n`);+ }+ await sh(['pnpm', 'install', '--no-frozen-lockfile']);+ const version = await resolvedTsClientVersion();+ if (version === STAINLESS_TS_VERSION) {+ fail(`override did not take effect: @composio/client still resolves to ${version}`);+ }+ console.log(`candidate @composio/client resolved: ${version}`);+ await sh(['pnpm', 'run', 'build:packages']);+};++const snapshotFiles = (paths) => new Map(paths.map((path) => [path, readFileSync(path)]));++const restoreFiles = (snapshot) => {+ for (const [path, contents] of snapshot) writeFileSync(path, contents);+};++const snapshotTsBaseline = () => snapshotFiles([WORKSPACE_YAML, LOCKFILE]);++const restoreTsBaseline = async (snapshot) => {+ restoreFiles(snapshot);+ await sh(['pnpm', 'install', '--frozen-lockfile']);+};++const verifyPyCandidate = async (wheel) => {+ if (!existsSync(wheel)) fail(`COMPOSIO_CLIENT_WHEEL not found: ${wheel}`);+ const out = await sh(['uv', 'run', '--project', 'python', '--with', wheel, 'python', '-c', 'import composio_client, importlib.metadata; print(importlib.metadata.version("composio-client"))']);+ const version = out.trim().split('\n').pop();+ if (!version.startsWith('2.')) {+ fail(`python candidate did not take effect (resolved ${version}); Stage B (SDK migration) must land first`);+ }+ return version;+};++// --- llm mock ---------------------------------------------------------------++/** Start aimock serving harness/llm-mock/fixtures; resolve once it answers. */+const startLlmMock = async (runDir) => {+ const mockDir = join(ROOT, 'harness', 'llm-mock');+ const logFile = join(runDir, 'aimock.log');+ const child = spawn(+ 'npx',+ ['-y', `@copilotkit/aimock@${AIMOCK_VERSION}`, '-c', 'aimock.json', '-p', String(AIMOCK_PORT), '-h', '127.0.0.1'],+ { cwd: mockDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }+ );+ child.stdout.on('data', (c) => appendFileSync(logFile, c));+ child.stderr.on('data', (c) => appendFileSync(logFile, c));++ // First run may download the package; allow a generous readiness window.+ const deadline = Date.now() + 120_000;+ for (;;) {+ try {+ const res = await fetch(`http://127.0.0.1:${AIMOCK_PORT}/v1/models`);+ if (res.ok) break;+ } catch { /* not up yet */ }+ if (Date.now() > deadline) {+ try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ }+ fail(`aimock did not become ready on :${AIMOCK_PORT} (see ${relative(ROOT, logFile)})`);+ }+ await new Promise((r) => setTimeout(r, 500));+ }+ console.log(`aimock ${AIMOCK_VERSION} ready on :${AIMOCK_PORT}`);+ return () => {+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }+ };+};++// --- commands ---------------------------------------------------------------++const cmdSweep = async () => {+ const client = opt('client', 'baseline');+ if (!['baseline', 'candidate'].includes(client)) fail(`--client must be baseline|candidate`);+ const llm = opt('llm', 'live');+ if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);+ LLM_MOCK = llm === 'mock';+ const entries = selectEntries();+ if (entries.length === 0) fail('no entries selected');+ if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);++ const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;+ const runDir = join(ARTIFACTS, id);+ mkdirSync(join(runDir, 'traces'), { recursive: true });++ let tsBaselineSnapshot;+ let stopLlmMock;+ const pyWheel = process.env.COMPOSIO_CLIENT_WHEEL;+ try {+ if (LLM_MOCK) stopLlmMock = await startLlmMock(runDir);+ if (client === 'candidate') {+ const wantTs = entries.some((e) => e.lang === 'ts');+ const wantPy = entries.some((e) => e.lang === 'py');+ if (wantTs) {+ if (!process.env.COMPOSIO_CLIENT_TARBALL) fail('candidate sweep with ts entries needs COMPOSIO_CLIENT_TARBALL');+ tsBaselineSnapshot = snapshotTsBaseline();+ await applyTsCandidate(process.env.COMPOSIO_CLIENT_TARBALL);+ }+ if (wantPy) {+ if (!pyWheel) fail('candidate sweep with py entries needs COMPOSIO_CLIENT_WHEEL');+ const v = await verifyPyCandidate(pyWheel);+ console.log(`candidate composio-client (py) resolved: ${v}`);+ for (const e of entries) if (e.lang === 'py') e.pyWith = [...(e.pyWith ?? []), pyWheel];+ }+ }++ console.log(`sweep ${id}: ${entries.length} entries → ${runDir}`);+ const results = await runPool(entries, runDir, { concurrency: Number(opt('concurrency', '4')) });+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);+ const counts = {+ green: results.filter((r) => r.status === 'green').length,+ red: results.filter((r) => r.status === 'red').length,+ skipped: results.filter((r) => r.status === 'skipped').length,+ };+ writeFileSync(+ join(runDir, 'summary.json'),+ JSON.stringify({ runId: id, client, llm, baseUrl: baseUrl(), counts, startedAt: new Date().toISOString() }, null, 2)+ );+ console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);+ process.exitCode = counts.red > 0 ? 1 : 0;+ } finally {+ if (stopLlmMock) stopLlmMock();+ if (tsBaselineSnapshot) await restoreTsBaseline(tsBaselineSnapshot);+ }+};++const cmdNeg = async () => {+ let entries = selectEntries().filter((e) => e.backend !== false);+ const sample = opt('sample');+ if (sample) {+ const rand = mulberry32(Number(opt('seed', '42')));+ entries = [...entries].sort(() => rand() - 0.5).slice(0, Number(sample));+ }+ const id = `${runId()}-neg`;+ const runDir = join(ARTIFACTS, id);+ mkdirSync(join(runDir, 'traces'), { recursive: true });+ console.log(`negative controls ${id}: ${entries.length} entries (garbage credentials; expectation: red)`);+ const results = await runPool(entries, runDir, { garbage: true, concurrency: Number(opt('concurrency', '6')) });+ const swallowers = results.filter((r) => r.status === 'green').map((r) => r.id);+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);+ writeFileSync(join(runDir, 'summary.json'), JSON.stringify({ runId: id, mode: 'neg', total: results.length, swallowers }, null, 2));+ if (swallowers.length > 0) {+ console.log(`NEGATIVE-CONTROL FAILURES (green under garbage credentials — these swallow errors):`);+ for (const s of swallowers) console.log(` ✗ ${s}`);+ process.exitCode = 1;+ } else {+ console.log('all negative controls red, as expected');+ }+};++const cmdSelftest = async () => {+ const runDir = join(ARTIFACTS, `${runId()}-selftest`);+ mkdirSync(join(runDir, 'traces'), { recursive: true });+ let failures = 0;+ const check = (name, ok) => {+ console.log(` ${ok ? '✓' : '✗'} ${name}`);+ if (!ok) failures += 1;+ };++ // 0. backend safety: the harness accepts only the canonical staging root.+ check('staging backend is accepted', requireStagingBaseUrl() === STAGING_BASE_URL);+ let productionRejected = false;+ try {+ requireStagingBaseUrl('https://backend.composio.dev');+ } catch {+ productionRejected = true;+ }+ check('production backend is rejected', productionRejected);++ let cleanupRan = false;+ let cleanupError;+ try {+ try {+ fail('expected selftest failure');+ } finally {+ cleanupRan = true;+ }+ } catch (error) {+ cleanupError = error;+ }+ check(+ 'usage failures unwind through cleanup',+ cleanupRan && cleanupError instanceof HarnessError+ );++ const cleanupFixture = join(runDir, 'cleanup-fixture.txt');+ writeFileSync(cleanupFixture, 'user contents');+ const cleanupSnapshot = snapshotFiles([cleanupFixture]);+ writeFileSync(cleanupFixture, 'candidate contents');+ restoreFiles(cleanupSnapshot);+ check('cleanup restores pre-run file contents', readFileSync(cleanupFixture, 'utf8') === 'user contents');++ // 1. known-good: error-handling-demo (no backend, no keys).+ const good = loadManifest().find((e) => e.id === 'ts/error-handling-demo/index');+ const { env: goodEnv } = buildEnv(good, runDir);+ delete goodEnv.COMPOSIO_API_KEY;+ const goodRes = await runEntry(good, goodEnv);+ check('known-good (error-handling-demo) exits 0', goodRes.exit === 0);++ // 2. known-bad fixtures: must exit non-zero in both languages.+ const badTsOut = await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/always-fails.ts']).then(() => 0).catch(() => 1);+ check('known-bad ts fixture exits non-zero', badTsOut === 1);+ const badPy = await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/always_fails.py']).then(() => 0).catch(() => 1);+ check('known-bad py fixture exits non-zero', badPy === 1);++ // 3. trace shims: an unauthenticated backend request must appear in the trace.+ const tsTrace = join(runDir, 'traces', 'shim-ts.jsonl');+ await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/trace-check.ts'], {+ env: { ...process.env, COMPOSIO_TRACE_FILE: tsTrace, COMPOSIO_BASE_URL: baseUrl(), NODE_OPTIONS: `--import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}` },+ }).catch(() => {});+ const tsLines = existsSync(tsTrace) ? readFileSync(tsTrace, 'utf8').trim().split('\n').filter(Boolean) : [];+ check('ts fetch shim recorded a composio call', tsLines.some((l) => JSON.parse(l).p?.includes('/api/')));++ const pyTrace = join(runDir, 'traces', 'shim-py.jsonl');+ await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/trace_check.py'], {+ env: { ...process.env, COMPOSIO_TRACE_FILE: pyTrace, COMPOSIO_BASE_URL: baseUrl(), PYTHONPATH: join(ROOT, 'harness', 'trace-py') },+ }).catch(() => {});+ const pyLines = existsSync(pyTrace) ? readFileSync(pyTrace, 'utf8').trim().split('\n').filter(Boolean) : [];+ check('py httpx shim recorded a composio call', pyLines.some((l) => JSON.parse(l).p?.includes('/api/')));++ // 4. parity must compare at least one entry and reject missing candidates.+ const parityBase = join(runDir, 'parity-base');+ const parityCandidate = join(runDir, 'parity-candidate');+ for (const dir of [parityBase, parityCandidate]) {+ mkdirSync(join(dir, 'traces'), { recursive: true });+ writeFileSync(join(dir, 'results.jsonl'), `${JSON.stringify({ id: 'fixture', status: 'green' })}\n`);+ writeFileSync(+ join(dir, 'traces', 'fixture.jsonl'),+ `${JSON.stringify({ m: 'GET', p: '/api/v3/toolkits', s: '2xx' })}\n`+ );+ }+ const parityMatches = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])+ .then(() => true)+ .catch(() => false);+ check('parity accepts matching runs', parityMatches);+ writeFileSync(join(parityCandidate, 'results.jsonl'), '');+ const missingCandidateRejected = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])+ .then(() => false)+ .catch(() => true);+ check('parity rejects missing candidate entries', missingCandidateRejected);++ rmSync(runDir, { recursive: true, force: true });+ console.log(failures === 0 ? 'selftest: PASS' : `selftest: FAIL (${failures})`);+ process.exitCode = failures === 0 ? 0 : 1;+};++try {+ switch (cmd) {+ case 'sweep':+ await cmdSweep();+ break;+ case 'neg':+ await cmdNeg();+ break;+ case 'selftest':+ await cmdSelftest();+ break;+ default:+ fail('usage: run.mjs <sweep|neg|selftest> [options]');+ }+} catch (error) {+ console.error(`run.mjs: ${error instanceof Error ? error.message : String(error)}`);+ process.exitCode = error instanceof HarnessError ? error.exitCode : 1;+}diff --git a/harness/staging-backend.mjs b/harness/staging-backend.mjsnew file mode 100644index 000000000..966e72469--- /dev/null+++ b/harness/staging-backend.mjs@@ -0,0 +1,25 @@+export const STAGING_BASE_URL = 'https://staging-backend.composio.dev';++export const requireStagingBaseUrl = (+ value = process.env.COMPOSIO_BASE_URL ?? STAGING_BASE_URL+) => {+ let url;+ try {+ url = new URL(value);+ } catch {+ throw new Error(`refusing invalid COMPOSIO_BASE_URL: ${value}`);+ }++ const isStagingRoot =+ url.origin === STAGING_BASE_URL &&+ url.pathname === '/' &&+ !url.search &&+ !url.hash &&+ !url.username &&+ !url.password;+ if (!isStagingRoot) {+ throw new Error(`refusing non-staging COMPOSIO_BASE_URL: ${value}`);+ }++ return STAGING_BASE_URL;+};diff --git a/harness/trace-py/sitecustomize.py b/harness/trace-py/sitecustomize.pynew file mode 100644index 000000000..634e323ab--- /dev/null+++ b/harness/trace-py/sitecustomize.py@@ -0,0 +1,108 @@+"""Harness-owned httpx tracer, loaded via PYTHONPATH (sitecustomize hook).++Examples must never reference COMPOSIO_TRACE_FILE themselves (lint-enforced).+"""++import json+import os+import re+import sys+from urllib.parse import urlsplit++_TRACE = os.environ.get("COMPOSIO_TRACE_FILE")+_STAGING_BASE_URL = "https://staging-backend.composio.dev"++if _TRACE:+ _base_url = os.environ.get("COMPOSIO_BASE_URL", _STAGING_BASE_URL)+ _parsed_base_url = urlsplit(_base_url)+ if not (+ _parsed_base_url.scheme == "https"+ and _parsed_base_url.hostname == "staging-backend.composio.dev"+ and _parsed_base_url.path in {"", "/"}+ and not _parsed_base_url.query+ and not _parsed_base_url.fragment+ and not _parsed_base_url.username+ and not _parsed_base_url.password+ ):+ print(+ f"sitecustomize: refusing non-staging COMPOSIO_BASE_URL: {_base_url}",+ file=sys.stderr,+ )+ os._exit(78)++ try:+ import httpx++ _LLM_HOSTS = {+ "api.openai.com",+ "api.anthropic.com",+ "generativelanguage.googleapis.com",+ }+ _BACKEND_HOST = _parsed_base_url.hostname+ _ID_SEG = re.compile(+ r"^(ca_|ac_|ti_|tr_|trs_|sess_|auth_|req_|proj_|org_)[\w-]+$"+ r"|^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"+ r"|^\d+$",+ re.IGNORECASE,+ )++ def _template(path: str) -> str:+ return "/".join("{id}" if _ID_SEG.match(seg) else seg for seg in path.split("/"))++ def _record(obj) -> None:+ try:+ with open(_TRACE, "a") as f:+ f.write(json.dumps(obj) + "\n")+ except OSError:+ pass++ def _log(request, response) -> None:+ host = request.url.host+ if host == _BACKEND_HOST:+ status = f"{response.status_code // 100}xx" if response is not None else "ERR"+ _record({"m": request.method.upper(), "p": _template(request.url.path), "s": status})+ elif host in _LLM_HOSTS:+ _record({"llm": host})++ # Outbound-email guard: tool executions matching the denylist are+ # refused at the transport, never forwarded to the backend.+ _DENY = re.compile(+ os.environ.get(+ "COMPOSIO_TOOL_DENYLIST",+ r"GMAIL_SEND|GMAIL_REPLY|SEND_EMAIL|SEND_DRAFT|OUTLOOK[A-Z_]*SEND",+ ),+ re.IGNORECASE,+ )++ def _guard(request) -> None:+ if request.url.host == _BACKEND_HOST and _DENY.search(request.url.path):+ _record({"m": request.method.upper(), "p": _template(request.url.path), "s": "BLOCKED"})+ raise RuntimeError(f"harness: outbound-email tool execution blocked ({request.url.path})")++ _orig_send = httpx.Client.send++ def _send(self, request, *args, **kwargs):+ _guard(request)+ response = None+ try:+ response = _orig_send(self, request, *args, **kwargs)+ return response+ finally:+ _log(request, response)++ httpx.Client.send = _send++ _orig_send_async = httpx.AsyncClient.send++ async def _send_async(self, request, *args, **kwargs):+ _guard(request)+ response = None+ try:+ response = await _orig_send_async(self, request, *args, **kwargs)+ return response+ finally:+ _log(request, response)++ httpx.AsyncClient.send = _send_async+ except Exception: # noqa: BLE001 - tracing must never break the example+ passdiff --git a/harness/trace/register.mjs b/harness/trace/register.mjsnew file mode 100644index 000000000..7c0e6adc6--- /dev/null+++ b/harness/trace/register.mjs@@ -0,0 +1,73 @@+// Harness-owned fetch tracer. Injected via NODE_OPTIONS=--import; examples+// must never reference COMPOSIO_TRACE_FILE themselves (lint-enforced).+import { requireStagingBaseUrl } from '../staging-backend.mjs';++const traceFile = process.env.COMPOSIO_TRACE_FILE;++if (traceFile) {+ const { appendFileSync } = await import('node:fs');++ const LLM_HOSTS = new Set([+ 'api.openai.com',+ 'api.anthropic.com',+ 'generativelanguage.googleapis.com',+ ]);++ const backendHost = new URL(requireStagingBaseUrl()).hostname;++ const ID_SEG =+ /^(ca_|ac_|ti_|tr_|trs_|sess_|auth_|req_|proj_|org_)[\w-]+$|^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$|^\d+$/i;+ const template = (path) =>+ path+ .split('/')+ .map((seg) => (ID_SEG.test(seg) ? '{id}' : seg))+ .join('/');++ const record = (obj) => {+ try {+ appendFileSync(traceFile, `${JSON.stringify(obj)}\n`);+ } catch {+ // never break the example over tracing+ }+ };++ // Outbound-email guard: tool executions matching the denylist are refused at+ // the transport, never forwarded to the backend.+ const DENY = new RegExp(process.env.COMPOSIO_TOOL_DENYLIST ?? 'GMAIL_SEND|GMAIL_REPLY|SEND_EMAIL|SEND_DRAFT|OUTLOOK[A-Z_]*SEND', 'i');++ const origFetch = globalThis.fetch;+ globalThis.fetch = async function tracedFetch(input, init) {+ let url;+ try {+ url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);+ } catch {+ return origFetch.call(this, input, init);+ }+ const method = (+ init?.method ??+ (typeof input === 'object' && input !== null && 'method' in input ? input.method : 'GET') ??+ 'GET'+ ).toUpperCase();++ if (url.hostname === backendHost && DENY.test(url.pathname)) {+ record({ m: method, p: template(url.pathname), s: 'BLOCKED' });+ throw new Error(`harness: outbound-email tool execution blocked (${url.pathname})`);+ }++ let res;+ try {+ res = await origFetch.call(this, input, init);+ return res;+ } finally {+ if (url.hostname === backendHost) {+ record({+ m: method,+ p: template(url.pathname),+ s: res ? `${Math.floor(res.status / 100)}xx` : 'ERR',+ });+ } else if (LLM_HOSTS.has(url.hostname)) {+ record({ llm: url.hostname });+ }+ }+ };+}diff --git a/parity-variance.json b/parity-variance.jsonnew file mode 100644index 000000000..046955d4b--- /dev/null+++ b/parity-variance.json@@ -0,0 +1,3 @@+{+ "entries": []+}diff --git a/scripts/examples-provision.mjs b/scripts/examples-provision.mjsindex 275955164..f62773c0c 100644--- a/scripts/examples-provision.mjs+++ b/scripts/examples-provision.mjs@@ -30,34 +30,23 @@// values are ever printed. The API-key demo value stored for serpapi is a// deliberately fake placeholder, not a real key.-const BASE_URL = process.env.COMPOSIO_BASE_URL ?? 'https://backend.composio.dev';+import { requireStagingBaseUrl } from '../harness/staging-backend.mjs';++let BASE_URL;+try {+ BASE_URL = requireStagingBaseUrl();+} catch (error) {+ console.error(error instanceof Error ? error.message : String(error));+ process.exit(1);+}const API_KEY = process.env.COMPOSIO_API_KEY;const USER_ID = process.env.COMPOSIO_EXAMPLES_USER_ID ?? 'examples';const INITIATE_MISSING = process.argv.includes('--initiate-missing');const GC = process.argv.includes('--gc');const DRY_RUN = process.argv.includes('--dry-run');-// The examples only ever run against the production backend with the-// disposable project key. Allowlist the Composio hosts rather than denylisting-// loopback spellings: the API key travels in every request, so anything that is-// not a Composio backend must be refused, not just localhost.-const { baseHost, baseProtocol } = (() => {- try {- const url = new URL(BASE_URL);- return { baseHost: url.hostname, baseProtocol: url.protocol };- } catch {- return { baseHost: null, baseProtocol: null };- }-})();-if (!baseHost || !(baseHost === 'composio.dev' || baseHost.endsWith('.composio.dev'))) {- console.error(`refusing non-Composio COMPOSIO_BASE_URL: ${BASE_URL}`);- process.exit(1);-}-if (baseProtocol !== 'https:') {- // The API key is sent on every request. Plain http would put it on the wire.- console.error(`refusing non-https COMPOSIO_BASE_URL: ${BASE_URL}`);- process.exit(1);-}+// The examples only ever run against staging with a disposable project key.+// requireStagingBaseUrl refuses every other host before the key is read or sent.if (!API_KEY) {console.error('COMPOSIO_API_KEY is required (dedicated examples-project key)');process.exit(1);