The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
ci: decouple codspeed from ci gate (#36051)
Mason Daughertya81203bf6a08
diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.ymlindex a86246f837..a53aa14f2e 100644--- a/.github/workflows/check_diffs.yml+++ b/.github/workflows/check_diffs.yml@@ -8,7 +8,6 @@# - Pydantic compatibility tests (_test_pydantic.yml)# - Integration test compilation checks (_compile_integration_test.yml)# - Extended test suites that require additional dependencies-# - Codspeed benchmarks (if not labeled 'codspeed-ignore')## Reports status to GitHub checks and PR status.@@ -67,7 +66,6 @@ jobs:compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }}dependencies: ${{ steps.set-matrix.outputs.dependencies }}test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }}- codspeed: ${{ steps.set-matrix.outputs.codspeed }}# Run linting only on packages that have changed fileslint:needs: [build]@@ -171,73 +169,6 @@ jobs:# and `set -e` above will cause the step to fail.echo "$STATUS" | grep 'nothing to commit, working tree clean'- # Run codspeed benchmarks only on packages that have changed files- codspeed:- name: "β‘ CodSpeed Benchmarks"- needs: [build]- if: ${{ needs.build.outputs.codspeed != '[]' && !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}- runs-on: ubuntu-latest- strategy:- matrix:- job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}- fail-fast: false- steps:- - uses: actions/checkout@v6-- - name: "π¦ Install UV Package Manager"- uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7- with:- # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+- # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106- python-version: "3.13.11"-- - uses: actions/setup-python@v6- with:- # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+- # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106- python-version: "3.13.11"-- - name: "π¦ Install Test Dependencies"- run: uv sync --group test- working-directory: ${{ matrix.job-configs.working-directory }}-- - name: "β‘ Run Benchmarks: ${{ matrix.job-configs.working-directory }}"- uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4- env:- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}- ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}- ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}- AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}- AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}- AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}- AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}- AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}- AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}- AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}- COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}- DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}- EXA_API_KEY: ${{ secrets.EXA_API_KEY }}- FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}- HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}- MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}- NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}- OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}- OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}- PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}- XAI_API_KEY: ${{ secrets.XAI_API_KEY }}- with:- token: ${{ secrets.CODSPEED_TOKEN }}- run: |- cd ${{ matrix.job-configs.working-directory }}- if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then- uv run --no-sync pytest ./tests/benchmarks --codspeed- else- uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed- fi- mode: ${{ matrix.job-configs.codspeed-mode }}-# Final status check - ensures all required jobs passed before allowing mergeci_success:name: "β CI Success"@@ -249,7 +180,6 @@ jobs:compile-integration-tests,extended-tests,test-pydantic,- codspeed,]if: |always()diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.ymlnew file mode 100644index 0000000000..10320299c0--- /dev/null+++ b/.github/workflows/codspeed.yml@@ -0,0 +1,109 @@+# CodSpeed performance benchmarks.+#+# Runs benchmarks on changed packages and uploads results to CodSpeed.+# Separated from the main CI workflow so that push-to-master baseline runs+# are never cancelled by subsequent merges (cancel-in-progress is only+# enabled for pull_request events).++name: "β‘ CodSpeed"++on:+ push:+ branches: [master]+ pull_request:++# On PRs, cancel stale runs when new commits are pushed.+# On push-to-master, never cancel β these runs populate CodSpeed baselines.+concurrency:+ group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++permissions:+ contents: read++env:+ UV_FROZEN: "true"+ UV_NO_SYNC: "true"++jobs:+ build:+ name: "Detect Changes"+ runs-on: ubuntu-latest+ if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}+ steps:+ - name: "π Checkout Code"+ uses: actions/checkout@v6+ - name: "π Setup Python 3.11"+ uses: actions/setup-python@v6+ with:+ python-version: "3.11"+ - name: "π Get Changed Files"+ id: files+ uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0+ - name: "π Analyze Changed Files"+ id: set-matrix+ run: |+ python -m pip install packaging requests+ python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT+ outputs:+ codspeed: ${{ steps.set-matrix.outputs.codspeed }}++ benchmarks:+ name: "β‘ CodSpeed Benchmarks"+ needs: [build]+ if: ${{ needs.build.outputs.codspeed != '[]' }}+ runs-on: ubuntu-latest+ strategy:+ matrix:+ job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}+ fail-fast: false+ steps:+ - uses: actions/checkout@v6++ - name: "π¦ Install UV Package Manager"+ uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7+ with:+ # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12++ # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106+ python-version: "3.13.11"++ - name: "π¦ Install Test Dependencies"+ run: uv sync --group test+ working-directory: ${{ matrix.job-configs.working-directory }}++ - name: "β‘ Run Benchmarks: ${{ matrix.job-configs.working-directory }}"+ uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4+ env:+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}+ ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}+ AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}+ AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}+ AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}+ AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}+ AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}+ AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}+ AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}+ COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}+ DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}+ EXA_API_KEY: ${{ secrets.EXA_API_KEY }}+ FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}+ HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}+ MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}+ NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}+ OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}+ PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}+ XAI_API_KEY: ${{ secrets.XAI_API_KEY }}+ with:+ token: ${{ secrets.CODSPEED_TOKEN }}+ run: |+ cd ${{ matrix.job-configs.working-directory }}+ if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then+ uv run --no-sync pytest ./tests/benchmarks --codspeed+ else+ uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed+ fi+ mode: ${{ matrix.job-configs.codspeed-mode }}