Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

ci: run private engine via GitHub App token (replaces public reusable-workflow call)

Soumya Medapaticeeac2ed004a
diff --git a/.github/workflows/docs-agent-eval.yml b/.github/workflows/docs-agent-eval.yml
index 89eef455f..e9f71ee5f 100644
--- a/.github/workflows/docs-agent-eval.yml
+++ b/.github/workflows/docs-agent-eval.yml
@@ -10,17 +10,16 @@
# - deployment_status (Vercel finished a docs preview): runs if the PR
# already carries a run-docs-tests label.
# - pull_request labeled/reopened: runs if the docs preview already
-# succeeded; otherwise skips, and the deployment_status event re-enters
-# when the preview lands. (`synchronize` removed: new pushes surface as
-# new deployments, so deployment_status covers them.)
+# succeeded; otherwise skips, and the deployment_status event re-enters.
#
-# All eval logic lives in ComposioHQ/docs-agent-eval-ci (reusable workflow) —
-# see its ARCHITECTURE.md. This check is informational and must NOT be made
-# a required status check in V1.
+# The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci.
+# A public repo cannot `uses:` a private repo's reusable workflow, so instead
+# a GitHub App token checks that repo out and this job runs it inline. See its
+# ARCHITECTURE.md. Informational check — must NOT be a required status in V1.
#
-# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY.
-# No Vercel credentials (decision 2026-08-19): the engine resolves previews
-# from GitHub's deployment records with the per-run token.
+# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY,
+# DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the
+# private engine repo).
name: Docs Agent Eval
on:
@@ -28,6 +27,11 @@ on:
types: [labeled, reopened]
deployment_status:
+env:
+ ENGINE_REPO: ComposioHQ/docs-agent-eval-ci
+ # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases.
+ ENGINE_REF: d299de8061151997548a3650e13ebfc63c03d08f
+
jobs:
# Decides IF an eval should start and FOR WHICH PR. deployment_status
# events know the commit but not the PR (deployments attach to commits),
@@ -114,27 +118,158 @@ jobs:
echo "mode=$mode"
} >> "$GITHUB_OUTPUT"
+ # Runs the eval inline: mint an App token, check out the private engine with
+ # it, then run the engine's code on this runner. (A public repo can't `uses:`
+ # a private reusable workflow, so the App token + checkout replaces it.)
eval:
needs: route
if: needs.route.outputs.should_run == 'true'
- # Concurrency here, after routing: a skipped/filtered event never enters
- # the group, so unrelated triggers cannot cancel a running eval. A newer
- # run for the same PR (new preview, re-label) still supersedes the stale
- # one.
concurrency:
group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}
cancel-in-progress: true
+ runs-on: ubuntu-latest
+ timeout-minutes: 55
permissions:
contents: read
- pull-requests: write # the result comment
- # Full-SHA pin required by org policy — bump on docs-agent-eval releases.
- uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@21146fbe43e6d6eb0db4b46ef9b09faab6110edd
- with:
- pr_number: ${{ needs.route.outputs.pr_number }}
- mode: ${{ needs.route.outputs.mode }}
- post_comment: true
- # Explicit and minimal on purpose — inherit would hand the engine every
- # secret this repo holds (review decision 2).
- secrets:
+ deployments: read
+ pull-requests: write # the result comment (posted with the built-in token)
+ env:
+ SOURCE_REPO: ${{ github.repository }}
+ PR_NUMBER: ${{ needs.route.outputs.pr_number }}
+ MODE: ${{ needs.route.outputs.mode }}
COMPOSIO_ORG_API_KEY: ${{ secrets.COMPOSIO_ORG_API_KEY }}
- DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+ # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint.
+ ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic
+ ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }}
+ DEEP_RUNS: '5'
+ POST_PR_COMMENT: 'true'
+ steps:
+ - name: Mint a GitHub App token (read the private engine repo)
+ id: app-token
+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+ with:
+ app-id: ${{ secrets.DOCS_EVAL_APP_ID }}
+ private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }}
+ owner: ComposioHQ
+ repositories: docs-agent-eval-ci
+
+ - name: Check out the private engine
+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ repository: ${{ env.ENGINE_REPO }}
+ ref: ${{ env.ENGINE_REF }}
+ token: ${{ steps.app-token.outputs.token }}
+ path: engine
+
+ - name: Setup Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+ with:
+ python-version: '3.12'
+ cache: pip
+ cache-dependency-path: engine/ci/requirements.txt
+
+ - name: Install harness dependencies
+ working-directory: engine
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -r ci/requirements.txt
+
+ - name: Cache npm downloads
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+ with:
+ path: ~/.npm
+ key: npm-claude-code-${{ runner.os }}
+
+ - name: Install Claude Code
+ run: npm install --global @anthropic-ai/claude-code
+
+ - name: Validate required credentials
+ shell: bash
+ run: |
+ test -n "$COMPOSIO_ORG_API_KEY" || { echo "COMPOSIO_ORG_API_KEY is not configured"; exit 1; }
+ test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; }
+
+ - name: Route (deterministic no-eval / Smoke / Deep)
+ id: extent
+ working-directory: engine
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ mkdir -p ci-output
+ python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json
+ cat ci-output/route.json
+ EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])')
+ HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])')
+ # run-docs-tests-deep forces Deep regardless of the router.
+ [ "$MODE" = Deep ] && EXTENT=deep
+ echo "extent=$EXTENT" >> "$GITHUB_OUTPUT"
+ echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT"
+
+ - name: Record no-eval outcome
+ if: steps.extent.outputs.extent == 'none'
+ working-directory: engine
+ shell: bash
+ run: |
+ python - <<'EOF'
+ import json
+ route = json.load(open("ci-output/route.json"))
+ json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2)
+ lines = "\n".join(f"- {r}" for r in route["reasons"])
+ open("ci-output/comment.md", "w").write(
+ f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n")
+ EOF
+
+ - name: Run exact-HEAD docs agent eval
+ if: steps.extent.outputs.extent != 'none'
+ continue-on-error: true
+ working-directory: engine
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ RUNMODE=Smoke
+ [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep
+ python ci/run_full_eval.py \
+ --repo "$SOURCE_REPO" \
+ --head-sha '${{ steps.extent.outputs.head_sha }}' \
+ --mode "$RUNMODE"
+
+ - name: Upload eval artifacts
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: docs-agent-eval-artifacts
+ path: engine/ci-output/
+ if-no-files-found: warn
+ retention-days: 14
+
+ - name: Publish job summary
+ if: always()
+ working-directory: engine
+ shell: bash
+ run: |
+ if [ -f ci-output/comment.md ]; then
+ cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY"
+ else
+ echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY"
+ echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Post result comment on the PR
+ if: always()
+ working-directory: engine
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ test -f ci-output/comment.md || exit 0
+ gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md
+
+ - name: Set check result
+ if: always()
+ working-directory: engine
+ shell: bash
+ run: |
+ test -f ci-output/outcome.json || exit 1
+ STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])')
+ test "$STATUS" = PASS || test "$STATUS" = NO_EVAL