Scrapy, a fast high-level web crawling & scraping framework for Python.
CI: Install dependencies with uv (#7838)
Adrian24de06bcd3bc
diff --git a/.github/dependabot.yml b/.github/dependabot.ymlnew file mode 100644index 000000000..623d48cfa--- /dev/null+++ b/.github/dependabot.yml@@ -0,0 +1,12 @@+version: 2+updates:+ - package-ecosystem: github-actions+ directory: "/"+ schedule:+ interval: monthly+ groups:+ github-actions:+ patterns:+ - "*"+ cooldown:+ default-days: 7diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.ymlindex 160b39488..15120b0d9 100644--- a/.github/workflows/auto-close-llm-pr.yml+++ b/.github/workflows/auto-close-llm-pr.yml@@ -1,5 +1,7 @@name: Auto-close LLM PRs-on:+# The workflow only reads the pull request body through the API, it never+# checks out or runs pull request code, so pull_request_target is safe here.+on: # zizmor: ignore[dangerous-triggers]pull_request_target:types: [opened]permissions:@@ -11,7 +13,7 @@ jobs:runs-on: ubuntu-lateststeps:- name: Check PR body and close if LLM-written- uses: actions/github-script@v6+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0with:github-token: ${{ secrets.GITHUB_TOKEN }}script: |diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex ed2388a59..331fade61 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -1,4 +1,8 @@name: Checks++permissions:+ contents: read+on:push:branches:@@ -13,6 +17,10 @@ concurrency:jobs:checks:runs-on: ubuntu-latest+ env:+ # Make uv use the interpreter that actions/setup-python installed instead+ # of downloading one of its own.+ UV_PYTHON_PREFERENCE: only-systemstrategy:fail-fast: falsematrix:@@ -38,21 +46,31 @@ jobs:TOXENV: twinechecksteps:- - uses: actions/checkout@v6+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v6+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: ${{ matrix.python-version }}+ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ docs/requirements.txt+ pyproject.toml+ tox.ini+- name: Run checkenv: ${{ matrix.env }}- run: |- pip install -U tox- tox+ run: uvx --with tox-uv toxpre-commit:runs-on: ubuntu-lateststeps:- - uses: actions/checkout@v6- - uses: pre-commit/[email protected]+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.ymlindex 6930c4c1c..82b16eea2 100644--- a/.github/workflows/codspeed.yml+++ b/.github/workflows/codspeed.yml@@ -22,24 +22,38 @@ permissions: {}jobs:benchmark:runs-on: ubuntu-latest+ env:+ # Make uv use the interpreter that actions/setup-python installed+ # instead of downloading one of its own.+ UV_PYTHON_PREFERENCE: only-systempermissions:contents: readid-token: write # OIDC authentication with CodSpeedsteps:- - uses: actions/checkout@v6+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- name: Set up Python 3.14- uses: actions/setup-python@v6+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: '3.14'++ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini+- name: Install dependencies+ # tox must stay on PATH for the CodSpeed action to invoke it.run: |- pip install --upgrade pip- pip install --upgrade tox+ uv tool install --with tox-uv toxtox -n -e benchmark- name: Run benchmarks- uses: CodSpeedHQ/action@v4+ uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1with:mode: simulationrun: tox -e benchmarkdiff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlindex 7779bbb6b..697647131 100644--- a/.github/workflows/publish.yml+++ b/.github/workflows/publish.yml@@ -1,4 +1,8 @@name: Publish++permissions:+ contents: read+on:push:tags:@@ -9,8 +13,28 @@ concurrency:cancel-in-progress: truejobs:+ build:+ name: Build distribution+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.14"+ - run: |+ python -m pip install --upgrade build+ python -m build+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: python-package-distributions+ path: dist/+publish:name: Upload release to PyPI+ needs:+ - buildruns-on: ubuntu-latestenvironment:name: pypi@@ -18,12 +42,9 @@ jobs:permissions:id-token: writesteps:- - uses: actions/checkout@v6- - uses: actions/setup-python@v6+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1with:- python-version: "3.14"- - run: |- python -m pip install --upgrade build- python -m build+ name: python-package-distributions+ path: dist/- name: Publish to PyPI- uses: pypa/gh-action-pypi-publish@release/v1+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.ymlindex 566b34e50..af2a0206a 100644--- a/.github/workflows/tests-macos.yml+++ b/.github/workflows/tests-macos.yml@@ -1,4 +1,8 @@name: macOS++permissions:+ contents: read+on:push:branches:@@ -15,6 +19,9 @@ jobs:runs-on: macos-latestenv:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}+ # Make uv use the interpreter that actions/setup-python installed instead+ # of downloading one of its own.+ UV_PYTHON_PREFERENCE: only-systemstrategy:fail-fast: falsematrix:@@ -31,25 +38,39 @@ jobs:TOXENV: no-reactorsteps:- - uses: actions/checkout@v6+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v6+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: ${{ matrix.python-version }}+ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini++ - name: Install mitmproxy+ env:+ # mitmproxy needs a newer Python than the oldest matrix entries, so let+ # uv download one where no system interpreter is new enough.+ UV_PYTHON_PREFERENCE: system+ run: uv tool install mitmproxy+- name: Run testsenv: ${{ matrix.env }}- run: |- pip install -U tox- tox+ run: uvx --with tox-uv tox- name: Upload coverage reportif: ${{ matrix.coverage }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0- name: Upload test resultsif: ${{ !cancelled() }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0with:report_type: test_resultsdiff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex ad2bcfce7..60a2bec21 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -1,4 +1,8 @@name: Ubuntu++permissions:+ contents: read+on:push:branches:@@ -15,6 +19,9 @@ jobs:runs-on: ubuntu-latestenv:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}+ # Make uv use the interpreter that actions/setup-python installed instead+ # of downloading one of its own.+ UV_PYTHON_PREFERENCE: only-systemstrategy:fail-fast: falsematrix:@@ -92,10 +99,12 @@ jobs:coverage: truesteps:- - uses: actions/checkout@v6+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v6+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: ${{ matrix.python-version }}@@ -105,21 +114,32 @@ jobs:sudo apt-get updatesudo apt-get install libxml2-dev libxslt-dev+ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini+- name: Install mitmproxy- run: pipx install mitmproxy+ env:+ # mitmproxy needs a newer Python than the oldest matrix entries, so let+ # uv download one where no system interpreter is new enough.+ UV_PYTHON_PREFERENCE: system+ # mitmproxy has no PyPy wheels, so run it on CPython regardless of the+ # interpreter under test.+ run: uv tool install --python cpython mitmproxy- name: Run testsenv: ${{ matrix.env }}- run: |- pip install -U tox- tox+ run: uvx --with tox-uv tox- name: Upload coverage reportif: ${{ matrix.coverage }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0- name: Upload test resultsif: ${{ !cancelled() }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0with:report_type: test_resultsdiff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex c33f96b12..884ff8e7c 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -1,4 +1,8 @@name: Windows++permissions:+ contents: read+on:push:branches:@@ -15,6 +19,9 @@ jobs:runs-on: windows-latestenv:PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}+ # Make uv use the interpreter that actions/setup-python installed instead+ # of downloading one of its own.+ UV_PYTHON_PREFERENCE: only-systemstrategy:fail-fast: falsematrix:@@ -55,25 +62,39 @@ jobs:TOXENV: extra-depssteps:- - uses: actions/checkout@v6+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v6+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0with:python-version: ${{ matrix.python-version }}+ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini++ - name: Install mitmproxy+ env:+ # mitmproxy needs a newer Python than the oldest matrix entries, so let+ # uv download one where no system interpreter is new enough.+ UV_PYTHON_PREFERENCE: system+ run: uv tool install mitmproxy+- name: Run testsenv: ${{ matrix.env }}- run: |- pip install -U tox- tox+ run: uvx --with tox-uv tox- name: Upload coverage reportif: ${{ matrix.coverage }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0- name: Upload test resultsif: ${{ !cancelled() }}- uses: codecov/codecov-action@v5+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0with:report_type: test_resultsdiff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex c27348c7a..c2cb5056d 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -27,6 +27,11 @@ repos:hooks:- id: sphinx-lint- repo: https://github.com/scrapy/sphinx-scrapy- rev: 0.8.9+ rev: 0.8.10hooks:- id: sphinx-scrapy+- repo: https://github.com/zizmorcore/zizmor-pre-commit+ rev: v1.28.0+ hooks:+ - id: zizmor+ args: [--no-progress, --fix]diff --git a/docs/requirements.in b/docs/requirements.inindex 257365380..3783dd1dc 100644--- a/docs/requirements.in+++ b/docs/requirements.in@@ -5,4 +5,4 @@ sphinxsphinx-notfound-pagesphinx-rtd-themesphinx-rtd-dark-mode-sphinx-scrapy @ git+https://github.com/scrapy/[email protected]+sphinx-scrapy @ git+https://github.com/scrapy/[email protected]diff --git a/docs/requirements.txt b/docs/requirements.txtindex 87634cea9..0f5969401 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0# via# -r docs/requirements.in# sphinx-rtd-dark-mode-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8# via -r docs/requirements.insphinx-sitemap==2.9.0# via sphinx-scrapydiff --git a/tox.ini b/tox.iniindex 8331e2ab6..edde83356 100644--- a/tox.ini+++ b/tox.ini@@ -5,7 +5,8 @@[tox]requires =- sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]+ sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]+ tox-uvenvlist =pre-commitpylint