Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
ci: parallelize change detection and gate style jobs (#41678)
Byron.wang81abb7322db4
diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.ymlindex 1b1afe96529..4e2abe4c433 100644--- a/.github/workflows/api-tests.yml+++ b/.github/workflows/api-tests.yml@@ -71,10 +71,10 @@ jobs:retention-days: 1api-integration:- name: API Integration Tests- runs-on: depot-ubuntu-24.04+ name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})+ runs-on: depot-ubuntu-24.04-4env:- COVERAGE_FILE: coverage-integration+ COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}STORAGE_TYPE: opendalOPENDAL_SCHEME: fsOPENDAL_FS_ROOT: /tmp/dify-storage@@ -82,9 +82,17 @@ jobs:run:shell: bashstrategy:+ fail-fast: falsematrix:python-version:- '3.12'+ shardIndex:+ - 1+ - 2+ - 3+ - 4+ shardTotal:+ - 4steps:- name: Checkout code@@ -112,6 +120,8 @@ jobs:-p no:benchmark \--start-middleware \-n auto \+ --shard-index "${{ matrix.shardIndex }}" \+ --shard-total "${{ matrix.shardTotal }}" \--timeout "${PYTEST_TIMEOUT:-180}" \api/tests/integration_tests/workflow \api/tests/integration_tests/tools \@@ -120,8 +130,9 @@ jobs:- name: Upload integration coverage datauses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1with:- name: api-coverage-integration- path: coverage-integration+ name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}+ path: ${{ env.COVERAGE_FILE }}+ if-no-files-found: errorretention-days: 1api-coverage:@@ -171,12 +182,15 @@ jobs:echo "" >> "$GITHUB_STEP_SUMMARY"unit_coverage="$(find coverage-data -type f -name coverage-unit -print -quit)"- integration_coverage="$(find coverage-data -type f -name coverage-integration -print -quit)"+ mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort): "${unit_coverage:?coverage-unit artifact not found}"- : "${integration_coverage:?coverage-integration artifact not found}"+ if [[ "${#integration_coverage[@]}" -ne 4 ]]; then+ echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2+ exit 1+ fireport_file="$(mktemp)"- uv run --project api coverage combine "$unit_coverage" "$integration_coverage"+ uv run --project api coverage combine "$unit_coverage" "${integration_coverage[@]}"uv run --project api coverage report --show-missing | tee "$report_file"echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"{diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.ymlindex 6023aa7fae2..b924fd18936 100644--- a/.github/workflows/main-ci.yml+++ b/.github/workflows/main-ci.yml@@ -37,8 +37,6 @@ jobs:# Check which paths were changed to determine which tests to runcheck-changes:name: Check Changed Files- needs: pre_job- if: needs.pre_job.outputs.should_skip != 'true'runs-on: depot-ubuntu-24.04outputs:api-changed: ${{ steps.changes.outputs.api }}@@ -49,6 +47,11 @@ jobs:migration-changed: ${{ steps.changes.outputs.migration }}sandbox-runtime-changed: ${{ steps.changes.outputs.sandbox-runtime }}dify-agent-changed: ${{ steps.changes.outputs.dify-agent }}+ python-style-changed: ${{ steps.changes.outputs.python-style }}+ dify-agent-style-changed: ${{ steps.changes.outputs.dify-agent-style }}+ web-style-changed: ${{ steps.changes.outputs.web-style }}+ ts-common-style-changed: ${{ steps.changes.outputs.ts-common-style }}+ superlinter-changed: ${{ steps.changes.outputs.superlinter }}steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3@@ -150,6 +153,58 @@ jobs:- 'docker/generate_docker_compose'- 'docker/ssrf_proxy/**'- 'docker/volumes/sandbox/conf/**'+ python-style:+ - 'api/**'+ - 'scripts/ast_grep_guard.py'+ - 'scripts/check_no_new_getattr.py'+ - 'scripts/check_no_new_controller_sqlalchemy.py'+ - 'scripts/lint_controller_sqlalchemy.py'+ - 'scripts/ast_grep_rules/no_new_getattr.yml'+ - 'scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml'+ - '.github/workflows/style.yml'+ - '.github/workflows/main-ci.yml'+ dify-agent-style:+ - 'dify-agent/**'+ - '.github/workflows/style.yml'+ - '.github/workflows/main-ci.yml'+ web-style:+ - 'web/**'+ - 'e2e/**'+ - 'sdks/nodejs-client/**'+ - 'packages/**'+ - 'package.json'+ - 'pnpm-lock.yaml'+ - 'pnpm-workspace.yaml'+ - 'knip.config.ts'+ - 'scripts/check-web-production-unused-after-knip-fix.mjs'+ - '.github/workflows/style.yml'+ - '.github/actions/setup-web/**'+ ts-common-style:+ - 'web/**'+ - 'cli/**'+ - 'e2e/**'+ - 'sdks/nodejs-client/**'+ - 'packages/**'+ - 'package.json'+ - 'pnpm-lock.yaml'+ - 'pnpm-workspace.yaml'+ - 'vite.config.ts'+ - 'lint.config.ts'+ - 'eslint.config.mjs'+ - 'knip.config.ts'+ - 'scripts/check-web-production-unused-after-knip-fix.mjs'+ - 'oxlint-suppressions.json'+ - 'eslint-suppressions.json'+ - '.vscode/**'+ - '.github/**'+ superlinter:+ - '**.sh'+ - '**.yaml'+ - '**.yml'+ - '**Dockerfile'+ - 'dev/**'+ - '.editorconfig'+ - '.vite-hooks/**'# Run tests in parallel while always emitting stable required checks.api-tests-run:@@ -396,10 +451,19 @@ jobs:style-check:name: Style Check- needs: pre_job+ if: ${{ always() }}+ needs:+ - pre_job+ - check-changesuses: ./.github/workflows/style.ymlwith:base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}+ classification-succeeded: ${{ needs.check-changes.result == 'success' }}+ run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}+ run-dify-agent-style: ${{ needs.check-changes.outputs.dify-agent-style-changed == 'true' }}+ run-web-style: ${{ needs.check-changes.outputs.web-style-changed == 'true' }}+ run-ts-common-style: ${{ needs.check-changes.outputs.ts-common-style-changed == 'true' }}+ run-superlinter: ${{ needs.check-changes.outputs.superlinter-changed == 'true' }}skip-checks: ${{ needs.pre_job.outputs.should_skip == 'true' }}vdb-tests-run:diff --git a/.github/workflows/style.yml b/.github/workflows/style.ymlindex d809f1e208b..11fb97b6963 100644--- a/.github/workflows/style.yml+++ b/.github/workflows/style.yml@@ -6,6 +6,30 @@ on:base-rev:required: truetype: string+ classification-succeeded:+ description: Whether the caller successfully classified the changed files.+ required: true+ type: boolean+ run-python-style:+ description: Run Python style checks for the changed files.+ required: true+ type: boolean+ run-dify-agent-style:+ description: Run Dify Agent style checks for the changed files.+ required: true+ type: boolean+ run-web-style:+ description: Run Web style checks for the changed files.+ required: true+ type: boolean+ run-ts-common-style:+ description: Run shared TypeScript style checks for the changed files.+ required: true+ type: boolean+ run-superlinter:+ description: Run SuperLinter for the changed files.+ required: true+ type: booleanskip-checks:description: Create the required check runs without repeating previously successful work.required: false@@ -20,42 +44,23 @@ permissions:jobs:python-style:name: Python Style- if: ${{ !inputs.skip-checks }}+ if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-python-style || inputs.run-dify-agent-style) }}runs-on: depot-ubuntu-24.04steps:+ - name: Verify changed-file classification+ if: ${{ !inputs.classification-succeeded }}+ run: |+ echo "Changed-file classification failed; refusing to skip Python style checks." >&2+ exit 1+- name: Checkout codeuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:persist-credentials: falsefetch-depth: 0- - name: Check changed files- id: changed-files- uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6- with:- files: |- api/**- scripts/ast_grep_guard.py- scripts/check_no_new_getattr.py- scripts/check_no_new_controller_sqlalchemy.py- scripts/lint_controller_sqlalchemy.py- scripts/ast_grep_rules/no_new_getattr.yml- scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml- .github/workflows/style.yml- .github/workflows/main-ci.yml-- - name: Check changed files (dify-agent)- id: changed-files-dify-agent- uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6- with:- files: |- dify-agent/**- .github/workflows/style.yml- .github/workflows/main-ci.yml-- name: Setup UV and Python- if: steps.changed-files.outputs.any_changed == 'true' || steps.changed-files-dify-agent.outputs.any_changed == 'true'uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1with:enable-cache: false@@ -65,46 +70,46 @@ jobs:dify-agent/uv.lock- name: Install dependencies- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv sync --project api --dev- name: Run Import Linter- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv run --directory api --dev lint-imports- name: Run Response Contract Linter- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv run --project api --dev python api/dev/lint_response_contracts.py --fail-on-mismatch- name: Run No New Getattr Guard- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv run --project api python scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"- name: Run No New Controller SQLAlchemy Guard- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv run --project api python scripts/check_no_new_controller_sqlalchemy.py --base-rev "${{ inputs.base-rev }}"- name: Run Type Checks- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-styleenv:PYREFLY_OUTPUT_FORMAT: githubrun: make type-check-core- name: Run Dify Agent Ruff- if: steps.changed-files-dify-agent.outputs.any_changed == 'true'+ if: inputs.run-dify-agent-stylerun: make -C dify-agent check- name: Run Dify Agent Type Checks- if: steps.changed-files-dify-agent.outputs.any_changed == 'true'+ if: inputs.run-dify-agent-stylerun: make -C dify-agent typecheck- name: Dotenv check- if: steps.changed-files.outputs.any_changed == 'true'+ if: inputs.run-python-stylerun: uv run --project api dotenv-linter ./api/.env.example ./web/.env.exampleweb-style:name: Web Style- if: ${{ !inputs.skip-checks }}+ if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-web-style) }}runs-on: depot-ubuntu-24.04defaults:run:@@ -114,126 +119,84 @@ jobs:pull-requests: readsteps:+ - name: Verify changed-file classification+ if: ${{ !inputs.classification-succeeded }}+ working-directory: .+ run: |+ echo "Changed-file classification failed; refusing to skip Web style checks." >&2+ exit 1+- name: Checkout codeuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:persist-credentials: false- - name: Check changed files- id: changed-files- uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6- with:- files: |- web/**- e2e/**- sdks/nodejs-client/**- packages/**- package.json- pnpm-lock.yaml- pnpm-workspace.yaml- knip.config.ts- scripts/check-web-production-unused-after-knip-fix.mjs- .github/workflows/style.yml- .github/actions/setup-web/**-- name: Setup web environment- if: steps.changed-files.outputs.any_changed == 'true'uses: ./.github/actions/setup-web- name: Web tsslint- if: steps.changed-files.outputs.any_changed == 'true'env:NODE_OPTIONS: --max-old-space-size=4096run: vp run lint:tss- name: Web dead code check- if: steps.changed-files.outputs.any_changed == 'true'working-directory: .run: vp run knip- name: Web dead code check production- if: steps.changed-files.outputs.any_changed == 'true'working-directory: .run: vp run knip:production- name: Web production unused declarations check- if: steps.changed-files.outputs.any_changed == 'true'working-directory: .run: vp run knip:production-unused-checkts-common-style:name: TS Common- if: ${{ !inputs.skip-checks }}+ if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-ts-common-style) }}runs-on: depot-ubuntu-24.04-4permissions:checks: writepull-requests: readsteps:+ - name: Verify changed-file classification+ if: ${{ !inputs.classification-succeeded }}+ run: |+ echo "Changed-file classification failed; refusing to skip TypeScript style checks." >&2+ exit 1+- name: Checkout codeuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:persist-credentials: false- - name: Check changed files- id: changed-files- uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6- with:- files: |- web/**- cli/**- e2e/**- sdks/nodejs-client/**- packages/**- package.json- pnpm-lock.yaml- pnpm-workspace.yaml- vite.config.ts- lint.config.ts- eslint.config.mjs- knip.config.ts- scripts/check-web-production-unused-after-knip-fix.mjs- oxlint-suppressions.json- eslint-suppressions.json- .vscode/**- .github/**-- name: Setup web environment- if: steps.changed-files.outputs.any_changed == 'true'uses: ./.github/actions/setup-web- name: Static check- if: steps.changed-files.outputs.any_changed == 'true'run: pnpm -w checksuperlinter:name: SuperLinter- if: ${{ !inputs.skip-checks }}+ if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-superlinter) }}runs-on: depot-ubuntu-24.04steps:+ - name: Verify changed-file classification+ if: ${{ !inputs.classification-succeeded }}+ run: |+ echo "Changed-file classification failed; refusing to skip SuperLinter." >&2+ exit 1+- name: Checkout codeuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:fetch-depth: 0persist-credentials: false- - name: Check changed files- id: changed-files- uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6- with:- files: |- **.sh- **.yaml- **.yml- **Dockerfile- dev/**- .editorconfig- .vite-hooks/**-- name: Super-linteruses: super-linter/super-linter/slim@4ce20838b8ab83717e78138c5b3a1407148e0918 # v8.7.0- if: steps.changed-files.outputs.any_changed == 'true'env:BASH_SEVERITY: warningDEFAULT_BRANCH: origin/maindiff --git a/api/conftest.py b/api/conftest.pyindex 350b0306016..e967a723d2a 100644--- a/api/conftest.py+++ b/api/conftest.py@@ -33,6 +33,18 @@ ensure_backend_test_environment(_REPO_ROOT)def pytest_addoption(parser: pytest.Parser) -> None:group = parser.getgroup("dify")+ group.addoption(+ "--shard-index",+ type=int,+ default=1,+ help="One-based index of the test shard to run.",+ )+ group.addoption(+ "--shard-total",+ type=int,+ default=1,+ help="Total number of test shards.",+ )group.addoption("--start-middleware",action="store_true",@@ -58,9 +70,33 @@ def pytest_addoption(parser: pytest.Parser) -> None:def pytest_configure(config: pytest.Config) -> None:+ shard_index = config.getoption("shard_index")+ shard_total = config.getoption("shard_total")+ if shard_total < 1:+ raise pytest.UsageError("--shard-total must be at least 1")+ if not 1 <= shard_index <= shard_total:+ raise pytest.UsageError("--shard-index must be between 1 and --shard-total")+config.stash[_DIFY_COMPOSE_STACKS_KEY] = []+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:+ """Select a deterministic, evenly sized slice of collected tests for this shard."""+ shard_index = config.getoption("shard_index")+ shard_total = config.getoption("shard_total")+ if shard_total == 1:+ return++ selected: list[pytest.Item] = []+ deselected: list[pytest.Item] = []+ for item_index, item in enumerate(items):+ target = selected if item_index % shard_total == shard_index - 1 else deselected+ target.append(item)++ config.hook.pytest_deselected(items=deselected)+ items[:] = selected++def pytest_sessionstart(session: pytest.Session) -> None:config = session.configif hasattr(config, "workerinput"):diff --git a/api/tests/unit_tests/commands/test_check_no_new_getattr.py b/api/tests/unit_tests/commands/test_check_no_new_getattr.pyindex a63569c706f..a846955a770 100644--- a/api/tests/unit_tests/commands/test_check_no_new_getattr.py+++ b/api/tests/unit_tests/commands/test_check_no_new_getattr.py@@ -228,19 +228,6 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None:assert checkout_step is not Noneassert "fetch-depth: 0" in checkout_step.group("step")- changed_files_step = re.search(- r"(?ms)^ - name: Check changed files\n.*?^ files: \|\n(?P<files>(?:^ \S[^\n]*\n)+)",- job_text,- )- assert changed_files_step is not None-- files_block = changed_files_step.group("files")- assert "api/**\n" in files_block- assert "scripts/check_no_new_getattr.py\n" in files_block- assert "scripts/ast_grep_rules/no_new_getattr.yml\n" in files_block- assert ".github/workflows/style.yml\n" in files_block- assert ".github/workflows/main-ci.yml\n" in files_block-guard_command = 'scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"'assert guard_command in job_text@@ -251,6 +238,7 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None:)assert guard_step is not None+ assert "if: inputs.run-python-style" in guard_step.group("step")assert "GITHUB_BASE_SHA" not in guard_step.group("step")@@ -266,17 +254,22 @@ def test_main_ci_passes_style_base_rev_input() -> None:"base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}"in style_job.group("job"))-- api_filter = re.search(- r"(?ms)^ api:\n(?P<filter>(?:^ - '[^']+'\n)+)",- workflow,- )- assert api_filter is not None- filter_text = api_filter.group("filter")- assert "scripts/check_no_new_getattr.py" in filter_text- assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text- assert ".github/workflows/style.yml" in filter_text- assert ".github/workflows/main-ci.yml" in filter_text+ assert "run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}" in style_job.group(+ "job"+ )++ for filter_name in ("api", "python-style"):+ path_filter = re.search(+ rf"(?ms)^ {re.escape(filter_name)}:\n(?P<filter>(?:^ - '[^']+'\n)+)",+ workflow,+ )+ assert path_filter is not None, filter_name+ filter_text = path_filter.group("filter")+ assert "api/**" in filter_text+ assert "scripts/check_no_new_getattr.py" in filter_text+ assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text+ assert ".github/workflows/style.yml" in filter_text+ assert ".github/workflows/main-ci.yml" in filter_textdef test_base_rev_mode_passes_when_only_legacy_getattr_exists(tmp_path: Path) -> None: