Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
ci: run private engine via GitHub App token (replaces public reusable-workflow call)
Soumya Medapaticeeac2ed004a
diff --git a/.github/workflows/docs-agent-eval.yml b/.github/workflows/docs-agent-eval.ymlindex 89eef455f..e9f71ee5f 100644--- a/.github/workflows/docs-agent-eval.yml+++ b/.github/workflows/docs-agent-eval.yml@@ -10,17 +10,16 @@# - deployment_status (Vercel finished a docs preview): runs if the PR# already carries a run-docs-tests label.# - pull_request labeled/reopened: runs if the docs preview already-# succeeded; otherwise skips, and the deployment_status event re-enters-# when the preview lands. (`synchronize` removed: new pushes surface as-# new deployments, so deployment_status covers them.)+# succeeded; otherwise skips, and the deployment_status event re-enters.#-# All eval logic lives in ComposioHQ/docs-agent-eval-ci (reusable workflow) —-# see its ARCHITECTURE.md. This check is informational and must NOT be made-# a required status check in V1.+# The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci.+# A public repo cannot `uses:` a private repo's reusable workflow, so instead+# a GitHub App token checks that repo out and this job runs it inline. See its+# ARCHITECTURE.md. Informational check — must NOT be a required status in V1.#-# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY.-# No Vercel credentials (decision 2026-08-19): the engine resolves previews-# from GitHub's deployment records with the per-run token.+# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY,+# DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the+# private engine repo).name: Docs Agent Evalon:@@ -28,6 +27,11 @@ on:types: [labeled, reopened]deployment_status:+env:+ ENGINE_REPO: ComposioHQ/docs-agent-eval-ci+ # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases.+ ENGINE_REF: d299de8061151997548a3650e13ebfc63c03d08f+jobs:# Decides IF an eval should start and FOR WHICH PR. deployment_status# events know the commit but not the PR (deployments attach to commits),@@ -114,27 +118,158 @@ jobs:echo "mode=$mode"} >> "$GITHUB_OUTPUT"+ # Runs the eval inline: mint an App token, check out the private engine with+ # it, then run the engine's code on this runner. (A public repo can't `uses:`+ # a private reusable workflow, so the App token + checkout replaces it.)eval:needs: routeif: needs.route.outputs.should_run == 'true'- # Concurrency here, after routing: a skipped/filtered event never enters- # the group, so unrelated triggers cannot cancel a running eval. A newer- # run for the same PR (new preview, re-label) still supersedes the stale- # one.concurrency:group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}cancel-in-progress: true+ runs-on: ubuntu-latest+ timeout-minutes: 55permissions:contents: read- pull-requests: write # the result comment- # Full-SHA pin required by org policy — bump on docs-agent-eval releases.- uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@21146fbe43e6d6eb0db4b46ef9b09faab6110edd- with:- pr_number: ${{ needs.route.outputs.pr_number }}- mode: ${{ needs.route.outputs.mode }}- post_comment: true- # Explicit and minimal on purpose — inherit would hand the engine every- # secret this repo holds (review decision 2).- secrets:+ deployments: read+ pull-requests: write # the result comment (posted with the built-in token)+ env:+ SOURCE_REPO: ${{ github.repository }}+ PR_NUMBER: ${{ needs.route.outputs.pr_number }}+ MODE: ${{ needs.route.outputs.mode }}COMPOSIO_ORG_API_KEY: ${{ secrets.COMPOSIO_ORG_API_KEY }}- DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}+ # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint.+ ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic+ ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }}+ DEEP_RUNS: '5'+ POST_PR_COMMENT: 'true'+ steps:+ - name: Mint a GitHub App token (read the private engine repo)+ id: app-token+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0+ with:+ app-id: ${{ secrets.DOCS_EVAL_APP_ID }}+ private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }}+ owner: ComposioHQ+ repositories: docs-agent-eval-ci++ - name: Check out the private engine+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ with:+ repository: ${{ env.ENGINE_REPO }}+ ref: ${{ env.ENGINE_REF }}+ token: ${{ steps.app-token.outputs.token }}+ path: engine++ - name: Setup Python+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5+ with:+ python-version: '3.12'+ cache: pip+ cache-dependency-path: engine/ci/requirements.txt++ - name: Install harness dependencies+ working-directory: engine+ run: |+ python -m pip install --upgrade pip+ python -m pip install -r ci/requirements.txt++ - name: Cache npm downloads+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4+ with:+ path: ~/.npm+ key: npm-claude-code-${{ runner.os }}++ - name: Install Claude Code+ run: npm install --global @anthropic-ai/claude-code++ - name: Validate required credentials+ shell: bash+ run: |+ test -n "$COMPOSIO_ORG_API_KEY" || { echo "COMPOSIO_ORG_API_KEY is not configured"; exit 1; }+ test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; }++ - name: Route (deterministic no-eval / Smoke / Deep)+ id: extent+ working-directory: engine+ shell: bash+ env:+ GITHUB_TOKEN: ${{ github.token }}+ run: |+ mkdir -p ci-output+ python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json+ cat ci-output/route.json+ EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])')+ HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])')+ # run-docs-tests-deep forces Deep regardless of the router.+ [ "$MODE" = Deep ] && EXTENT=deep+ echo "extent=$EXTENT" >> "$GITHUB_OUTPUT"+ echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT"++ - name: Record no-eval outcome+ if: steps.extent.outputs.extent == 'none'+ working-directory: engine+ shell: bash+ run: |+ python - <<'EOF'+ import json+ route = json.load(open("ci-output/route.json"))+ json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2)+ lines = "\n".join(f"- {r}" for r in route["reasons"])+ open("ci-output/comment.md", "w").write(+ f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n")+ EOF++ - name: Run exact-HEAD docs agent eval+ if: steps.extent.outputs.extent != 'none'+ continue-on-error: true+ working-directory: engine+ env:+ GITHUB_TOKEN: ${{ github.token }}+ run: |+ RUNMODE=Smoke+ [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep+ python ci/run_full_eval.py \+ --repo "$SOURCE_REPO" \+ --head-sha '${{ steps.extent.outputs.head_sha }}' \+ --mode "$RUNMODE"++ - name: Upload eval artifacts+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: docs-agent-eval-artifacts+ path: engine/ci-output/+ if-no-files-found: warn+ retention-days: 14++ - name: Publish job summary+ if: always()+ working-directory: engine+ shell: bash+ run: |+ if [ -f ci-output/comment.md ]; then+ cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY"+ else+ echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY"+ echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY"+ fi++ - name: Post result comment on the PR+ if: always()+ working-directory: engine+ shell: bash+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ test -f ci-output/comment.md || exit 0+ gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md++ - name: Set check result+ if: always()+ working-directory: engine+ shell: bash+ run: |+ test -f ci-output/outcome.json || exit 1+ STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])')+ test "$STATUS" = PASS || test "$STATUS" = NO_EVAL