Skip to content

The agent engineering platform.

ci: decouple codspeed from ci gate (#36051)

Mason Daughertya81203bf6a08
diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml
index a86246f837..a53aa14f2e 100644
--- a/.github/workflows/check_diffs.yml
+++ b/.github/workflows/check_diffs.yml
@@ -8,7 +8,6 @@
# - Pydantic compatibility tests (_test_pydantic.yml)
# - Integration test compilation checks (_compile_integration_test.yml)
# - Extended test suites that require additional dependencies
-# - Codspeed benchmarks (if not labeled 'codspeed-ignore')
#
# Reports status to GitHub checks and PR status.
@@ -67,7 +66,6 @@ jobs:
compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }}
dependencies: ${{ steps.set-matrix.outputs.dependencies }}
test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }}
- codspeed: ${{ steps.set-matrix.outputs.codspeed }}
# Run linting only on packages that have changed files
lint:
needs: [build]
@@ -171,73 +169,6 @@ jobs:
# and `set -e` above will cause the step to fail.
echo "$STATUS" | grep 'nothing to commit, working tree clean'
- # Run codspeed benchmarks only on packages that have changed files
- codspeed:
- name: "⚑ CodSpeed Benchmarks"
- needs: [build]
- if: ${{ needs.build.outputs.codspeed != '[]' && !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}
- fail-fast: false
- steps:
- - uses: actions/checkout@v6
-
- - name: "πŸ“¦ Install UV Package Manager"
- uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7
- with:
- # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+
- # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106
- python-version: "3.13.11"
-
- - uses: actions/setup-python@v6
- with:
- # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+
- # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106
- python-version: "3.13.11"
-
- - name: "πŸ“¦ Install Test Dependencies"
- run: uv sync --group test
- working-directory: ${{ matrix.job-configs.working-directory }}
-
- - name: "⚑ Run Benchmarks: ${{ matrix.job-configs.working-directory }}"
- uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4
- env:
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}
- ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}
- AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
- AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
- AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
- AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
- AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
- AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
- AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
- COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
- DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
- EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
- FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
- MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
- NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
- OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}
- XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
- with:
- token: ${{ secrets.CODSPEED_TOKEN }}
- run: |
- cd ${{ matrix.job-configs.working-directory }}
- if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
- uv run --no-sync pytest ./tests/benchmarks --codspeed
- else
- uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed
- fi
- mode: ${{ matrix.job-configs.codspeed-mode }}
-
# Final status check - ensures all required jobs passed before allowing merge
ci_success:
name: "βœ… CI Success"
@@ -249,7 +180,6 @@ jobs:
compile-integration-tests,
extended-tests,
test-pydantic,
- codspeed,
]
if: |
always()
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
new file mode 100644
index 0000000000..10320299c0
--- /dev/null
+++ b/.github/workflows/codspeed.yml
@@ -0,0 +1,109 @@
+# CodSpeed performance benchmarks.
+#
+# Runs benchmarks on changed packages and uploads results to CodSpeed.
+# Separated from the main CI workflow so that push-to-master baseline runs
+# are never cancelled by subsequent merges (cancel-in-progress is only
+# enabled for pull_request events).
+
+name: "⚑ CodSpeed"
+
+on:
+ push:
+ branches: [master]
+ pull_request:
+
+# On PRs, cancel stale runs when new commits are pushed.
+# On push-to-master, never cancel β€” these runs populate CodSpeed baselines.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+ contents: read
+
+env:
+ UV_FROZEN: "true"
+ UV_NO_SYNC: "true"
+
+jobs:
+ build:
+ name: "Detect Changes"
+ runs-on: ubuntu-latest
+ if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
+ steps:
+ - name: "πŸ“‹ Checkout Code"
+ uses: actions/checkout@v6
+ - name: "🐍 Setup Python 3.11"
+ uses: actions/setup-python@v6
+ with:
+ python-version: "3.11"
+ - name: "πŸ“‚ Get Changed Files"
+ id: files
+ uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0
+ - name: "πŸ” Analyze Changed Files"
+ id: set-matrix
+ run: |
+ python -m pip install packaging requests
+ python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
+ outputs:
+ codspeed: ${{ steps.set-matrix.outputs.codspeed }}
+
+ benchmarks:
+ name: "⚑ CodSpeed Benchmarks"
+ needs: [build]
+ if: ${{ needs.build.outputs.codspeed != '[]' }}
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}
+ fail-fast: false
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: "πŸ“¦ Install UV Package Manager"
+ uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7
+ with:
+ # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+
+ # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106
+ python-version: "3.13.11"
+
+ - name: "πŸ“¦ Install Test Dependencies"
+ run: uv sync --group test
+ working-directory: ${{ matrix.job-configs.working-directory }}
+
+ - name: "⚑ Run Benchmarks: ${{ matrix.job-configs.working-directory }}"
+ uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4
+ env:
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}
+ ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}
+ AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+ AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+ AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
+ AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
+ AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
+ AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
+ COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+ DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+ EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
+ FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+ HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+ MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
+ NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
+ OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+ PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}
+ XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
+ with:
+ token: ${{ secrets.CODSPEED_TOKEN }}
+ run: |
+ cd ${{ matrix.job-configs.working-directory }}
+ if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
+ uv run --no-sync pytest ./tests/benchmarks --codspeed
+ else
+ uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed
+ fi
+ mode: ${{ matrix.job-configs.codspeed-mode }}