The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
infra: create individual jobs in check_diff, do max milvus testing in 3.11 (#23829)
Erick Friis9de562f747cd
diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.pyindex ed0cab169c..c3b2559303 100644--- a/.github/scripts/check_diff.py+++ b/.github/scripts/check_diff.py@@ -53,6 +53,44 @@ def add_dependents(dirs_to_eval: Set[str], dependents: dict) -> List[str]:return list(updated)+def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:+ min_python = "3.8"+ max_python = "3.12"++ # custom logic for specific directories+ if dir_ == "libs/partners/milvus":+ # milvus poetry doesn't allow 3.12 because they+ # declare deps in funny way+ max_python = "3.11"++ return [+ {"working-directory": dir_, "python-version": min_python},+ {"working-directory": dir_, "python-version": max_python},+ ]+++def _get_configs_for_multi_dirs(+ job: str, dirs_to_run: List[str], dependents: dict+) -> List[Dict[str, str]]:+ if job == "lint":+ dirs = add_dependents(+ dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],+ dependents,+ )+ elif job in ["test", "compile-integration-tests", "dependencies"]:+ dirs = add_dependents(+ dirs_to_run["test"] | dirs_to_run["extended-test"], dependents+ )+ elif job == "extended-tests":+ dirs = list(dirs_to_run["extended-test"])+ else:+ raise ValueError(f"Unknown job: {job}")++ return [+ config for dir_ in dirs for config in _get_configs_for_single_dir(job, dir_)+ ]++if __name__ == "__main__":files = sys.argv[1:]@@ -126,17 +164,23 @@ if __name__ == "__main__":dependents = dependents_graph()- outputs = {- "dirs-to-lint": add_dependents(- dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],- dependents,- ),- "dirs-to-test": add_dependents(- dirs_to_run["test"] | dirs_to_run["extended-test"], dependents- ),- "dirs-to-extended-test": list(dirs_to_run["extended-test"]),- "docs-edited": "true" if docs_edited else "",+ # we now have dirs_by_job+ # todo: clean this up++ map_job_to_configs = {+ job: _get_configs_for_multi_dirs(job, dirs_to_run, dependents)+ for job in [+ "lint",+ "test",+ "extended-tests",+ "compile-integration-tests",+ "dependencies",+ ]}- for key, value in outputs.items():+ map_job_to_configs["test-doc-imports"] = (+ [{"python-version": "3.12"}] if docs_edited else []+ )++ for key, value in map_job_to_configs.items():json_output = json.dumps(value)print(f"{key}={json_output}")diff --git a/.github/workflows/_compile_integration_test.yml b/.github/workflows/_compile_integration_test.ymlindex d8dc998715..491151ac57 100644--- a/.github/workflows/_compile_integration_test.yml+++ b/.github/workflows/_compile_integration_test.yml@@ -7,6 +7,10 @@ on:required: truetype: stringdescription: "From which folder this pipeline executes"+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -25,14 +29,14 @@ jobs:- "3.10"- "3.11"- "3.12"- name: "poetry run pytest -m compile tests/integration_tests #${{ matrix.python-version }}"+ name: "poetry run pytest -m compile tests/integration_tests #${{ inputs.python-version }}"steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}working-directory: ${{ inputs.working-directory }}cache-key: compile-integrationdiff --git a/.github/workflows/_dependencies.yml b/.github/workflows/_dependencies.ymlindex cbc9caf16e..b96fe04ed0 100644--- a/.github/workflows/_dependencies.yml+++ b/.github/workflows/_dependencies.yml@@ -11,6 +11,10 @@ on:required: falsetype: stringdescription: "Relative path to the langchain library folder"+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -21,22 +25,14 @@ jobs:run:working-directory: ${{ inputs.working-directory }}runs-on: ubuntu-latest- strategy:- matrix:- python-version:- - "3.8"- - "3.9"- - "3.10"- - "3.11"- - "3.12"- name: dependency checks ${{ matrix.python-version }}+ name: dependency checks ${{ inputs.python-version }}steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}working-directory: ${{ inputs.working-directory }}cache-key: pydantic-cross-compatdiff --git a/.github/workflows/_integration_test.yml b/.github/workflows/_integration_test.ymlindex 700a8d1014..05971e1dfc 100644--- a/.github/workflows/_integration_test.yml+++ b/.github/workflows/_integration_test.yml@@ -6,6 +6,10 @@ on:working-directory:required: truetype: string+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -16,19 +20,14 @@ jobs:run:working-directory: ${{ inputs.working-directory }}runs-on: ubuntu-latest- strategy:- matrix:- python-version:- - "3.8"- - "3.11"- name: Python ${{ matrix.python-version }}+ name: Python ${{ inputs.python-version }}steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}working-directory: ${{ inputs.working-directory }}cache-key: corediff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.ymlindex e48f360ddc..264dfa6eb8 100644--- a/.github/workflows/_lint.yml+++ b/.github/workflows/_lint.yml@@ -11,6 +11,10 @@ on:required: falsetype: stringdescription: "Relative path to the langchain library folder"+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -21,27 +25,15 @@ env:jobs:build:- name: "make lint #${{ matrix.python-version }}"+ name: "make lint #${{ inputs.python-version }}"runs-on: ubuntu-latest- strategy:- matrix:- # Only lint on the min and max supported Python versions.- # It's extremely unlikely that there's a lint issue on any version in between- # that doesn't show up on the min or max versions.- #- # GitHub rate-limits how many jobs can be running at any one time.- # Starting new jobs is also relatively slow,- # so linting on fewer versions makes CI faster.- python-version:- - "3.8"- - "3.12"steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}working-directory: ${{ inputs.working-directory }}cache-key: lint-with-extras@@ -86,7 +78,7 @@ jobs:with:path: |${{ env.WORKDIR }}/.mypy_cache- key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}+ key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ inputs.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}- name: Analysing the code with our lint@@ -120,7 +112,7 @@ jobs:with:path: |${{ env.WORKDIR }}/.mypy_cache_test- key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}+ key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ inputs.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}- name: Analysing the code with our lintworking-directory: ${{ inputs.working-directory }}diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.ymlindex ee04fb5e23..841f3796f3 100644--- a/.github/workflows/_test.yml+++ b/.github/workflows/_test.yml@@ -11,6 +11,10 @@ on:required: falsetype: stringdescription: "Relative path to the langchain library folder"+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -21,22 +25,14 @@ jobs:run:working-directory: ${{ inputs.working-directory }}runs-on: ubuntu-latest- strategy:- matrix:- python-version:- - "3.8"- - "3.9"- - "3.10"- - "3.11"- - "3.12"- name: "make test #${{ matrix.python-version }}"+ name: "make test #${{ inputs.python-version }}"steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}working-directory: ${{ inputs.working-directory }}cache-key: corediff --git a/.github/workflows/_test_doc_imports.yml b/.github/workflows/_test_doc_imports.ymlindex cd2827903b..17a92a5933 100644--- a/.github/workflows/_test_doc_imports.yml+++ b/.github/workflows/_test_doc_imports.yml@@ -2,6 +2,11 @@ name: test_doc_importson:workflow_call:+ inputs:+ python-version:+ required: true+ type: string+ description: "Python version to use"env:POETRY_VERSION: "1.7.1"@@ -13,14 +18,14 @@ jobs:matrix:python-version:- "3.12"- name: "check doc imports #${{ matrix.python-version }}"+ name: "check doc imports #${{ inputs.python-version }}"steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ inputs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}cache-key: corediff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.ymlindex 33d31bad7f..0149f5ec11 100644--- a/.github/workflows/check_diffs.yml+++ b/.github/workflows/check_diffs.yml@@ -33,91 +33,96 @@ jobs:run: |python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUToutputs:- dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }}- dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }}- dirs-to-extended-test: ${{ steps.set-matrix.outputs.dirs-to-extended-test }}- docs-edited: ${{ steps.set-matrix.outputs.docs-edited }}+ lint: ${{ steps.set-matrix.outputs.lint }}+ test: ${{ steps.set-matrix.outputs.test }}+ extended-tests: ${{ steps.set-matrix.outputs.extended-tests }}+ compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }}+ dependencies: ${{ steps.set-matrix.outputs.dependencies }}+ test-doc-imports: ${{ steps.set-matrix.outputs.test-doc-imports }}lint:- name: cd ${{ matrix.working-directory }}+ name: cd ${{ matrix.job-configs.working-directory }}needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-lint != '[]' }}+ if: ${{ needs.build.outputs.lint != '[]' }}strategy:matrix:- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-lint) }}+ job-configs: ${{ fromJson(needs.build.outputs.lint) }}uses: ./.github/workflows/_lint.ymlwith:- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}+ python-version: ${{ matrix.job-configs.python-version }}secrets: inherittest:- name: cd ${{ matrix.working-directory }}+ name: cd ${{ matrix.job-configs.working-directory }}needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}+ if: ${{ needs.build.outputs.test != '[]' }}strategy:matrix:- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}+ job-configs: ${{ fromJson(needs.build.outputs.test) }}uses: ./.github/workflows/_test.ymlwith:- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}+ python-version: ${{ matrix.job-configs.python-version }}secrets: inherittest-doc-imports:needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-test != '[]' || needs.build.outputs.docs-edited }}+ if: ${{ needs.build.outputs.test-doc-imports != '[]' }}+ strategy:+ matrix:+ job-configs: ${{ fromJson(needs.build.outputs.test-doc-imports) }}uses: ./.github/workflows/_test_doc_imports.ymlsecrets: inherit+ with:+ python-version: ${{ matrix.job-configs.python-version }}compile-integration-tests:- name: cd ${{ matrix.working-directory }}+ name: cd ${{ matrix.job-configs.working-directory }}needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}+ if: ${{ needs.build.outputs.compile-integration-tests != '[]' }}strategy:matrix:- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}+ job-configs: ${{ fromJson(needs.build.outputs.compile-integration-tests) }}uses: ./.github/workflows/_compile_integration_test.ymlwith:- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}+ python-version: ${{ matrix.job-configs.python-version }}secrets: inheritdependencies:- name: cd ${{ matrix.working-directory }}+ name: cd ${{ matrix.job-configs.working-directory }}needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}+ if: ${{ needs.build.outputs.dependencies != '[]' }}strategy:matrix:- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}+ job-configs: ${{ fromJson(needs.build.outputs.dependencies) }}uses: ./.github/workflows/_dependencies.ymlwith:- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}+ python-version: ${{ matrix.job-configs.python-version }}secrets: inheritextended-tests:- name: "cd ${{ matrix.working-directory }} / make extended_tests #${{ matrix.python-version }}"+ name: "cd ${{ matrix.job-configs.working-directory }} / make extended_tests #${{ matrix.job-configs.python-version }}"needs: [ build ]- if: ${{ needs.build.outputs.dirs-to-extended-test != '[]' }}+ if: ${{ needs.build.outputs.extended-tests != '[]' }}strategy:matrix:# note different variable for extended test dirs- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-extended-test) }}- python-version:- - "3.8"- - "3.9"- - "3.10"- - "3.11"- - "3.12"+ job-configs: ${{ fromJson(needs.build.outputs.extended-tests) }}runs-on: ubuntu-latestdefaults:run:- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}steps:- uses: actions/checkout@v4- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}+ - name: Set up Python ${{ matrix.job-configs.python-version }} + Poetry ${{ env.POETRY_VERSION }}uses: "./.github/actions/poetry_setup"with:- python-version: ${{ matrix.python-version }}+ python-version: ${{ matrix.job-configs.python-version }}poetry-version: ${{ env.POETRY_VERSION }}- working-directory: ${{ matrix.working-directory }}+ working-directory: ${{ matrix.job-configs.working-directory }}cache-key: extended- name: Install dependencies