Skip to content

The agent engineering platform.

infra: create individual jobs in check_diff, do max milvus testing in 3.11 (#23829)

Erick Friis9de562f747cd
diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py
index ed0cab169c..c3b2559303 100644
--- a/.github/scripts/check_diff.py
+++ b/.github/scripts/check_diff.py
@@ -53,6 +53,44 @@ def add_dependents(dirs_to_eval: Set[str], dependents: dict) -> List[str]:
return list(updated)
+def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
+ min_python = "3.8"
+ max_python = "3.12"
+
+ # custom logic for specific directories
+ if dir_ == "libs/partners/milvus":
+ # milvus poetry doesn't allow 3.12 because they
+ # declare deps in funny way
+ max_python = "3.11"
+
+ return [
+ {"working-directory": dir_, "python-version": min_python},
+ {"working-directory": dir_, "python-version": max_python},
+ ]
+
+
+def _get_configs_for_multi_dirs(
+ job: str, dirs_to_run: List[str], dependents: dict
+) -> List[Dict[str, str]]:
+ if job == "lint":
+ dirs = add_dependents(
+ dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
+ dependents,
+ )
+ elif job in ["test", "compile-integration-tests", "dependencies"]:
+ dirs = add_dependents(
+ dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
+ )
+ elif job == "extended-tests":
+ dirs = list(dirs_to_run["extended-test"])
+ else:
+ raise ValueError(f"Unknown job: {job}")
+
+ return [
+ config for dir_ in dirs for config in _get_configs_for_single_dir(job, dir_)
+ ]
+
+
if __name__ == "__main__":
files = sys.argv[1:]
@@ -126,17 +164,23 @@ if __name__ == "__main__":
dependents = dependents_graph()
- outputs = {
- "dirs-to-lint": add_dependents(
- dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"],
- dependents,
- ),
- "dirs-to-test": add_dependents(
- dirs_to_run["test"] | dirs_to_run["extended-test"], dependents
- ),
- "dirs-to-extended-test": list(dirs_to_run["extended-test"]),
- "docs-edited": "true" if docs_edited else "",
+ # we now have dirs_by_job
+ # todo: clean this up
+
+ map_job_to_configs = {
+ job: _get_configs_for_multi_dirs(job, dirs_to_run, dependents)
+ for job in [
+ "lint",
+ "test",
+ "extended-tests",
+ "compile-integration-tests",
+ "dependencies",
+ ]
}
- for key, value in outputs.items():
+ map_job_to_configs["test-doc-imports"] = (
+ [{"python-version": "3.12"}] if docs_edited else []
+ )
+
+ for key, value in map_job_to_configs.items():
json_output = json.dumps(value)
print(f"{key}={json_output}")
diff --git a/.github/workflows/_compile_integration_test.yml b/.github/workflows/_compile_integration_test.yml
index d8dc998715..491151ac57 100644
--- a/.github/workflows/_compile_integration_test.yml
+++ b/.github/workflows/_compile_integration_test.yml
@@ -7,6 +7,10 @@ on:
required: true
type: string
description: "From which folder this pipeline executes"
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -25,14 +29,14 @@ jobs:
- "3.10"
- "3.11"
- "3.12"
- name: "poetry run pytest -m compile tests/integration_tests #${{ matrix.python-version }}"
+ name: "poetry run pytest -m compile tests/integration_tests #${{ inputs.python-version }}"
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: compile-integration
diff --git a/.github/workflows/_dependencies.yml b/.github/workflows/_dependencies.yml
index cbc9caf16e..b96fe04ed0 100644
--- a/.github/workflows/_dependencies.yml
+++ b/.github/workflows/_dependencies.yml
@@ -11,6 +11,10 @@ on:
required: false
type: string
description: "Relative path to the langchain library folder"
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -21,22 +25,14 @@ jobs:
run:
working-directory: ${{ inputs.working-directory }}
runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version:
- - "3.8"
- - "3.9"
- - "3.10"
- - "3.11"
- - "3.12"
- name: dependency checks ${{ matrix.python-version }}
+ name: dependency checks ${{ inputs.python-version }}
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: pydantic-cross-compat
diff --git a/.github/workflows/_integration_test.yml b/.github/workflows/_integration_test.yml
index 700a8d1014..05971e1dfc 100644
--- a/.github/workflows/_integration_test.yml
+++ b/.github/workflows/_integration_test.yml
@@ -6,6 +6,10 @@ on:
working-directory:
required: true
type: string
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -16,19 +20,14 @@ jobs:
run:
working-directory: ${{ inputs.working-directory }}
runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version:
- - "3.8"
- - "3.11"
- name: Python ${{ matrix.python-version }}
+ name: Python ${{ inputs.python-version }}
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: core
diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml
index e48f360ddc..264dfa6eb8 100644
--- a/.github/workflows/_lint.yml
+++ b/.github/workflows/_lint.yml
@@ -11,6 +11,10 @@ on:
required: false
type: string
description: "Relative path to the langchain library folder"
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -21,27 +25,15 @@ env:
jobs:
build:
- name: "make lint #${{ matrix.python-version }}"
+ name: "make lint #${{ inputs.python-version }}"
runs-on: ubuntu-latest
- strategy:
- matrix:
- # Only lint on the min and max supported Python versions.
- # It's extremely unlikely that there's a lint issue on any version in between
- # that doesn't show up on the min or max versions.
- #
- # GitHub rate-limits how many jobs can be running at any one time.
- # Starting new jobs is also relatively slow,
- # so linting on fewer versions makes CI faster.
- python-version:
- - "3.8"
- - "3.12"
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: lint-with-extras
@@ -86,7 +78,7 @@ jobs:
with:
path: |
${{ env.WORKDIR }}/.mypy_cache
- key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}
+ key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ inputs.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}
- name: Analysing the code with our lint
@@ -120,7 +112,7 @@ jobs:
with:
path: |
${{ env.WORKDIR }}/.mypy_cache_test
- key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}
+ key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ inputs.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}
- name: Analysing the code with our lint
working-directory: ${{ inputs.working-directory }}
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index ee04fb5e23..841f3796f3 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -11,6 +11,10 @@ on:
required: false
type: string
description: "Relative path to the langchain library folder"
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -21,22 +25,14 @@ jobs:
run:
working-directory: ${{ inputs.working-directory }}
runs-on: ubuntu-latest
- strategy:
- matrix:
- python-version:
- - "3.8"
- - "3.9"
- - "3.10"
- - "3.11"
- - "3.12"
- name: "make test #${{ matrix.python-version }}"
+ name: "make test #${{ inputs.python-version }}"
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: core
diff --git a/.github/workflows/_test_doc_imports.yml b/.github/workflows/_test_doc_imports.yml
index cd2827903b..17a92a5933 100644
--- a/.github/workflows/_test_doc_imports.yml
+++ b/.github/workflows/_test_doc_imports.yml
@@ -2,6 +2,11 @@ name: test_doc_imports
on:
workflow_call:
+ inputs:
+ python-version:
+ required: true
+ type: string
+ description: "Python version to use"
env:
POETRY_VERSION: "1.7.1"
@@ -13,14 +18,14 @@ jobs:
matrix:
python-version:
- "3.12"
- name: "check doc imports #${{ matrix.python-version }}"
+ name: "check doc imports #${{ inputs.python-version }}"
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ inputs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ inputs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
cache-key: core
diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml
index 33d31bad7f..0149f5ec11 100644
--- a/.github/workflows/check_diffs.yml
+++ b/.github/workflows/check_diffs.yml
@@ -33,91 +33,96 @@ jobs:
run: |
python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
outputs:
- dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }}
- dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }}
- dirs-to-extended-test: ${{ steps.set-matrix.outputs.dirs-to-extended-test }}
- docs-edited: ${{ steps.set-matrix.outputs.docs-edited }}
+ lint: ${{ steps.set-matrix.outputs.lint }}
+ test: ${{ steps.set-matrix.outputs.test }}
+ extended-tests: ${{ steps.set-matrix.outputs.extended-tests }}
+ compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }}
+ dependencies: ${{ steps.set-matrix.outputs.dependencies }}
+ test-doc-imports: ${{ steps.set-matrix.outputs.test-doc-imports }}
lint:
- name: cd ${{ matrix.working-directory }}
+ name: cd ${{ matrix.job-configs.working-directory }}
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-lint != '[]' }}
+ if: ${{ needs.build.outputs.lint != '[]' }}
strategy:
matrix:
- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-lint) }}
+ job-configs: ${{ fromJson(needs.build.outputs.lint) }}
uses: ./.github/workflows/_lint.yml
with:
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
+ python-version: ${{ matrix.job-configs.python-version }}
secrets: inherit
test:
- name: cd ${{ matrix.working-directory }}
+ name: cd ${{ matrix.job-configs.working-directory }}
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
+ if: ${{ needs.build.outputs.test != '[]' }}
strategy:
matrix:
- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+ job-configs: ${{ fromJson(needs.build.outputs.test) }}
uses: ./.github/workflows/_test.yml
with:
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
+ python-version: ${{ matrix.job-configs.python-version }}
secrets: inherit
test-doc-imports:
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-test != '[]' || needs.build.outputs.docs-edited }}
+ if: ${{ needs.build.outputs.test-doc-imports != '[]' }}
+ strategy:
+ matrix:
+ job-configs: ${{ fromJson(needs.build.outputs.test-doc-imports) }}
uses: ./.github/workflows/_test_doc_imports.yml
secrets: inherit
+ with:
+ python-version: ${{ matrix.job-configs.python-version }}
compile-integration-tests:
- name: cd ${{ matrix.working-directory }}
+ name: cd ${{ matrix.job-configs.working-directory }}
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
+ if: ${{ needs.build.outputs.compile-integration-tests != '[]' }}
strategy:
matrix:
- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+ job-configs: ${{ fromJson(needs.build.outputs.compile-integration-tests) }}
uses: ./.github/workflows/_compile_integration_test.yml
with:
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
+ python-version: ${{ matrix.job-configs.python-version }}
secrets: inherit
dependencies:
- name: cd ${{ matrix.working-directory }}
+ name: cd ${{ matrix.job-configs.working-directory }}
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-test != '[]' }}
+ if: ${{ needs.build.outputs.dependencies != '[]' }}
strategy:
matrix:
- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }}
+ job-configs: ${{ fromJson(needs.build.outputs.dependencies) }}
uses: ./.github/workflows/_dependencies.yml
with:
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
+ python-version: ${{ matrix.job-configs.python-version }}
secrets: inherit
extended-tests:
- name: "cd ${{ matrix.working-directory }} / make extended_tests #${{ matrix.python-version }}"
+ name: "cd ${{ matrix.job-configs.working-directory }} / make extended_tests #${{ matrix.job-configs.python-version }}"
needs: [ build ]
- if: ${{ needs.build.outputs.dirs-to-extended-test != '[]' }}
+ if: ${{ needs.build.outputs.extended-tests != '[]' }}
strategy:
matrix:
# note different variable for extended test dirs
- working-directory: ${{ fromJson(needs.build.outputs.dirs-to-extended-test) }}
- python-version:
- - "3.8"
- - "3.9"
- - "3.10"
- - "3.11"
- - "3.12"
+ job-configs: ${{ fromJson(needs.build.outputs.extended-tests) }}
runs-on: ubuntu-latest
defaults:
run:
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
steps:
- uses: actions/checkout@v4
- - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+ - name: Set up Python ${{ matrix.job-configs.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
- python-version: ${{ matrix.python-version }}
+ python-version: ${{ matrix.job-configs.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
- working-directory: ${{ matrix.working-directory }}
+ working-directory: ${{ matrix.job-configs.working-directory }}
cache-key: extended
- name: Install dependencies