Skip to content

The agent engineering platform.

ci: use partial runs w/ codspeed (#32920)

Sydney Runkle0814bfe5eddd
diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml
index 33a17c9780..667eda3dd2 100644
--- a/.github/workflows/check_diffs.yml
+++ b/.github/workflows/check_diffs.yml
@@ -54,6 +54,7 @@ jobs:
dependencies: ${{ steps.set-matrix.outputs.dependencies }}
test-doc-imports: ${{ steps.set-matrix.outputs.test-doc-imports }}
test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }}
+ codspeed: ${{ steps.set-matrix.outputs.codspeed }}
# Run linting only on packages that have changed files
lint:
needs: [ build ]
@@ -166,10 +167,50 @@ jobs:
# and `set -e` above will cause the step to fail.
echo "$STATUS" | grep 'nothing to commit, working tree clean'
+ # Run codspeed benchmarks only on packages that have changed files
+ codspeed:
+ name: '⚑ CodSpeed Benchmarks'
+ needs: [ build ]
+ if: ${{ needs.build.outputs.codspeed != '[]' && !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}
+ fail-fast: false
+ steps:
+ - uses: actions/checkout@v5
+
+ # We have to use 3.12 as 3.13 is not yet supported
+ - name: 'πŸ“¦ Install UV Package Manager'
+ uses: astral-sh/setup-uv@v6
+ with:
+ python-version: "3.12"
+
+ - uses: actions/setup-python@v6
+ with:
+ python-version: "3.12"
+
+ - name: 'πŸ“¦ Install Test Dependencies'
+ run: uv sync --group test
+ working-directory: ${{ matrix.job-configs.working-directory }}
+
+ - name: '⚑ Run Benchmarks: ${{ matrix.job-configs.working-directory }}'
+ uses: CodSpeedHQ/action@v3
+ with:
+ token: ${{ secrets.CODSPEED_TOKEN }}
+ run: |
+ cd ${{ matrix.job-configs.working-directory }}
+ if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
+ uv run --no-sync pytest ./tests/benchmarks --codspeed
+ else
+ uv run --no-sync pytest ./tests/ --codspeed
+ fi
+ mode: ${{ matrix.job-configs.working-directory == 'libs/core' && 'walltime' || 'instrumentation' }}
+
# Final status check - ensures all required jobs passed before allowing merge
ci_success:
name: 'βœ… CI Success'
- needs: [build, lint, test, compile-integration-tests, extended-tests, test-doc-imports, test-pydantic]
+ needs: [build, lint, test, compile-integration-tests, extended-tests, test-doc-imports, test-pydantic, codspeed]
if: |
always()
runs-on: ubuntu-latest
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
deleted file mode 100644
index ae942ea9c9..0000000000
--- a/.github/workflows/codspeed.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-name: '⚑ CodSpeed'
-
-on:
- push:
- branches:
- - master
- pull_request:
- workflow_dispatch:
-
-permissions:
- contents: read
-
-env:
- AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: foo
- AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: foo
- DEEPSEEK_API_KEY: foo
- FIREWORKS_API_KEY: foo
-
-jobs:
- codspeed:
- name: 'Benchmark'
- runs-on: ubuntu-latest
- if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
- strategy:
- matrix:
- include:
- - working-directory: libs/core
- mode: walltime
- - working-directory: libs/partners/openai
- - working-directory: libs/partners/anthropic
- - working-directory: libs/partners/deepseek
- - working-directory: libs/partners/fireworks
- - working-directory: libs/partners/xai
- - working-directory: libs/partners/mistralai
- - working-directory: libs/partners/groq
- fail-fast: false
-
- steps:
- - uses: actions/checkout@v5
-
- # We have to use 3.12 as 3.13 is not yet supported
- - name: 'πŸ“¦ Install UV Package Manager'
- uses: astral-sh/setup-uv@v6
- with:
- python-version: "3.12"
-
- - uses: actions/setup-python@v6
- with:
- python-version: "3.12"
-
- - name: 'πŸ“¦ Install Test Dependencies'
- run: uv sync --group test
- working-directory: ${{ matrix.working-directory }}
-
- - name: '⚑ Run Benchmarks: ${{ matrix.working-directory }}'
- uses: CodSpeedHQ/action@v3
- with:
- token: ${{ secrets.CODSPEED_TOKEN }}
- run: |
- cd ${{ matrix.working-directory }}
- if [ "${{ matrix.working-directory }}" = "libs/core" ]; then
- uv run --no-sync pytest ./tests/benchmarks --codspeed
- else
- uv run --no-sync pytest ./tests/ --codspeed
- fi
- mode: ${{ matrix.mode || 'instrumentation' }}