Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
ci: dual-trigger docs eval, org-member gate, no Vercel secrets
Soumya Medapati6fa60f203efb
diff --git a/.github/workflows/docs-agent-eval.yml b/.github/workflows/docs-agent-eval.ymlindex ede382d3a..4a67a8d04 100644--- a/.github/workflows/docs-agent-eval.yml+++ b/.github/workflows/docs-agent-eval.yml@@ -6,49 +6,135 @@# the proposed docs through a rewrite proxy, and the result is posted back# here as a PR comment. `run-docs-tests-deep` forces Deep.#-# All eval logic lives in ComposioHQ/docs-agent-eval (reusable workflow) —+# Two entry paths, so label-then-push and push-then-label both work:+# - deployment_status (Vercel finished a docs preview): runs if the PR+# already carries a run-docs-tests label.+# - pull_request labeled/reopened: runs if the docs preview already+# succeeded; otherwise skips, and the deployment_status event re-enters+# when the preview lands. (`synchronize` removed: new pushes surface as+# new deployments, so deployment_status covers them.)+#+# All eval logic lives in ComposioHQ/docs-agent-eval-ci (reusable workflow) —# see its ARCHITECTURE.md. This check is informational and must NOT be made# a required status check in V1.#-# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY,-# VERCEL_PROJECT_ID,-# VERCEL_TOKEN (see the PR description for provisioning notes).+# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY.+# No Vercel credentials (decision 2026-08-19): the engine resolves previews+# from GitHub's deployment records with the per-run token.name: Docs Agent Evalon:pull_request:- types: [labeled, synchronize, reopened]+ types: [labeled, reopened]+ deployment_status:jobs:- eval:- # Run when the PR carries the label — but for `labeled` events, only when- # the label just added is one of ours (any other label addition would- # otherwise cancel-and-restart an in-flight eval; Cursor Bugbot finding).- # Either label triggers on its own (run-docs-tests-deep standalone forces- # Deep); labeled events still only count when the added label is ours.+ # Decides IF an eval should start and FOR WHICH PR. deployment_status+ # events know the commit but not the PR (deployments attach to commits),+ # so this job maps SHA -> open PR before any label check is possible.+ route:if: |- (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') ||- contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) &&- (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))- # Job-level on purpose: a skipped job never enters the concurrency group,- # so unrelated triggers cannot cancel a running eval. New commits still- # supersede stale runs.+ (github.event_name == 'pull_request' &&+ (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') ||+ contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) &&+ (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))) ||+ (github.event_name == 'deployment_status' &&+ github.event.deployment_status.state == 'success' &&+ contains(github.event.deployment_status.environment_url, '.preview.composio.dev'))+ runs-on: ubuntu-latest+ timeout-minutes: 5+ permissions:+ contents: read+ deployments: read+ pull-requests: read+ outputs:+ should_run: ${{ steps.decide.outputs.should_run }}+ pr_number: ${{ steps.decide.outputs.pr_number }}+ mode: ${{ steps.decide.outputs.mode }}+ steps:+ - name: Decide whether to run, and for which PR+ id: decide+ shell: bash+ env:+ GH_TOKEN: ${{ github.token }}+ REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ pr="" assoc="" labels="" preview_ok=false++ if [ "$GITHUB_EVENT_NAME" = deployment_status ]; then+ # Trap 2: the event payload has no PR number — map commit -> PR.+ SHA=$(jq -r .deployment.sha "$GITHUB_EVENT_PATH")+ PRJSON=$(gh api "repos/$REPO/commits/$SHA/pulls" \+ --jq '[.[] | select(.state=="open")][0] // empty')+ if [ -n "$PRJSON" ]; then+ pr=$(jq -r .number <<<"$PRJSON")+ assoc=$(jq -r .author_association <<<"$PRJSON")+ labels=$(jq -r '[.labels[].name] | join(",")' <<<"$PRJSON")+ fi+ preview_ok=true # this event IS the successful preview+ else+ pr=$(jq -r .pull_request.number "$GITHUB_EVENT_PATH")+ assoc=$(jq -r .pull_request.author_association "$GITHUB_EVENT_PATH")+ labels=$(jq -r '[.pull_request.labels[].name] | join(",")' "$GITHUB_EVENT_PATH")+ SHA=$(jq -r .pull_request.head.sha "$GITHUB_EVENT_PATH")+ # Trap 1: label may predate the preview. Only run now if a docs+ # preview already succeeded for this exact commit; otherwise skip+ # — the deployment_status event re-enters when the preview lands.+ for d in $(gh api "repos/$REPO/deployments?sha=$SHA&per_page=10" --jq '.[].id'); do+ URL=$(gh api "repos/$REPO/deployments/$d/statuses" \+ --jq '[.[] | select(.state=="success")][0].environment_url // empty')+ case "$URL" in+ https://docs-*.preview.composio.dev*) preview_ok=true; break ;;+ esac+ done+ fi++ haslabel=false+ case ",$labels," in+ *,run-docs-tests,*|*,run-docs-tests-deep,*) haslabel=true ;;+ esac++ # Employee gate: PR author must be org-affiliated (public repo).+ trusted=false+ case "$assoc" in MEMBER|OWNER|COLLABORATOR) trusted=true ;; esac++ mode=Auto+ case ",$labels," in *,run-docs-tests-deep,*) mode=Deep ;; esac++ should=false+ if [ -n "$pr" ] && $haslabel && $trusted && $preview_ok; then+ should=true+ fi+ echo "event=$GITHUB_EVENT_NAME pr=${pr:-none} labels=[$labels] assoc=${assoc:-none} preview_ok=$preview_ok -> should_run=$should"++ {+ echo "should_run=$should"+ echo "pr_number=$pr"+ echo "mode=$mode"+ } >> "$GITHUB_OUTPUT"++ eval:+ needs: route+ if: needs.route.outputs.should_run == 'true'+ # Concurrency here, after routing: a skipped/filtered event never enters+ # the group, so unrelated triggers cannot cancel a running eval. A newer+ # run for the same PR (new preview, re-label) still supersedes the stale+ # one.concurrency:- group: docs-agent-eval-pr-${{ github.event.pull_request.number }}+ group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}cancel-in-progress: truepermissions:contents: readpull-requests: write # the result comment# Full-SHA pin required by org policy — bump on docs-agent-eval releases.- uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@48e1a1d2f8baf12a7a073b8141afede3baee94f7+ uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@2a831429ee6d8be6b9b4526f78bf5a451881cf8fwith:- pr_number: ${{ github.event.pull_request.number }}- mode: ${{ contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep') && 'Deep' || 'Auto' }}+ pr_number: ${{ needs.route.outputs.pr_number }}+ mode: ${{ needs.route.outputs.mode }}post_comment: true# Explicit and minimal on purpose — inherit would hand the engine every# secret this repo holds (review decision 2).secrets:COMPOSIO_ORG_API_KEY: ${{ secrets.COMPOSIO_ORG_API_KEY }}DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}- VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}- VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}