The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
benchmarks: always run (not conditional on changes) (#31409)
Sydney Runkle1917dd1ccd74
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.ymlindex 55cb0d8f75..7fd69eb1be 100644--- a/.github/workflows/codspeed.yml+++ b/.github/workflows/codspeed.yml@@ -5,10 +5,6 @@ on:branches:- masterpull_request:- paths:- - 'libs/core/**'- - 'libs/partners/**'-workflow_dispatch:env:@@ -20,31 +16,21 @@ env:FIREWORKS_API_KEY: foojobs:- prepare_matrix:- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v4- - name: Install uv- uses: astral-sh/setup-uv@v6- with:- python-version: "3.12"- - id: files- uses: Ana06/[email protected]- - id: set-matrix- run: |- uv venv- uv pip install packaging requests- uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT- outputs:- codspeed: ${{ steps.set-matrix.outputs.codspeed }}codspeed:name: Run benchmarks- needs: [ prepare_matrix ]- if: ${{ needs.prepare_matrix.outputs.codspeed != '[]' }}runs-on: ubuntu-lateststrategy:matrix:- job-configs: ${{ fromJson(needs.prepare_matrix.outputs.codspeed) }}+ include:+ - working-directory: libs/core+ mode: walltime+ - working-directory: libs/partners/openai+ - working-directory: libs/partners/anthropic+ - working-directory: libs/partners/deepseek+ - working-directory: libs/partners/fireworks+ - working-directory: libs/partners/xai+ - working-directory: libs/partners/mistralai+ - working-directory: libs/partners/groqfail-fast: falsesteps:@@ -54,26 +40,26 @@ jobs:- name: Install uvuses: astral-sh/setup-uv@v6with:- python-version: ${{ matrix.job-configs.python-version }}+ python-version: "3.12"- uses: actions/setup-python@v5with:- python-version: ${{ matrix.job-configs.python-version }}+ python-version: "3.12"- name: Install dependenciesrun: uv sync --group test- working-directory: ${{ matrix.job-configs.working-directory }}+ working-directory: ${{ matrix.working-directory }}- - name: Run benchmarks ${{ matrix.job-configs.working-directory }}+ - name: Run benchmarks ${{ matrix.working-directory }}uses: CodSpeedHQ/action@v3with:token: ${{ secrets.CODSPEED_TOKEN }}run: |- cd ${{ matrix.job-configs.working-directory }}- if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then+ cd ${{ matrix.working-directory }}+ if [ "${{ matrix.working-directory }}" = "libs/core" ]; thenuv run --no-sync pytest ./tests/benchmarks --codspeedelseuv run --no-sync pytest ./tests/ --codspeedfi- mode: ${{ matrix.job-configs.working-directory == 'libs/core' && 'walltime' || 'instrumentation' }}+ mode: ${{ matrix.mode || 'instrumentation' }}diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.pyindex f58cabcb61..2b7ad0faf5 100644--- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py@@ -1012,4 +1012,9 @@ class ChatModelUnitTests(ChatModelTests):"""Test initialization time of the chat model. If this test fails, check thatwe are not introducing undue overhead in the model's initialization."""- _ = benchmark(self.chat_model_class, **self.chat_model_params)++ def _init_in_loop() -> None:+ for _ in range(10):+ self.chat_model_class(**self.chat_model_params)++ benchmark(_init_in_loop)