Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CI: Install dependencies with uv (#7838)

Adrian24de06bcd3bc
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..623d48cfa
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+updates:
+ - package-ecosystem: github-actions
+ directory: "/"
+ schedule:
+ interval: monthly
+ groups:
+ github-actions:
+ patterns:
+ - "*"
+ cooldown:
+ default-days: 7
diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml
index 160b39488..15120b0d9 100644
--- a/.github/workflows/auto-close-llm-pr.yml
+++ b/.github/workflows/auto-close-llm-pr.yml
@@ -1,5 +1,7 @@
name: Auto-close LLM PRs
-on:
+# The workflow only reads the pull request body through the API, it never
+# checks out or runs pull request code, so pull_request_target is safe here.
+on: # zizmor: ignore[dangerous-triggers]
pull_request_target:
types: [opened]
permissions:
@@ -11,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check PR body and close if LLM-written
- uses: actions/github-script@v6
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index ed2388a59..331fade61 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -1,4 +1,8 @@
name: Checks
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -13,6 +17,10 @@ concurrency:
jobs:
checks:
runs-on: ubuntu-latest
+ env:
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@@ -38,21 +46,31 @@ jobs:
TOXENV: twinecheck
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v6
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ docs/requirements.txt
+ pyproject.toml
+ tox.ini
+
- name: Run check
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
pre-commit:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v6
- - uses: pre-commit/[email protected]
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index 6930c4c1c..82b16eea2 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -22,24 +22,38 @@ permissions: {}
jobs:
benchmark:
runs-on: ubuntu-latest
+ env:
+ # Make uv use the interpreter that actions/setup-python installed
+ # instead of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
permissions:
contents: read
id-token: write # OIDC authentication with CodSpeed
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python 3.14
- uses: actions/setup-python@v6
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
+
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
- name: Install dependencies
+ # tox must stay on PATH for the CodSpeed action to invoke it.
run: |
- pip install --upgrade pip
- pip install --upgrade tox
+ uv tool install --with tox-uv tox
tox -n -e benchmark
- name: Run benchmarks
- uses: CodSpeedHQ/action@v4
+ uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
with:
mode: simulation
run: tox -e benchmark
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 7779bbb6b..697647131 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -1,4 +1,8 @@
name: Publish
+
+permissions:
+ contents: read
+
on:
push:
tags:
@@ -9,8 +13,28 @@ concurrency:
cancel-in-progress: true
jobs:
+ build:
+ name: Build distribution
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.14"
+ - run: |
+ python -m pip install --upgrade build
+ python -m build
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: python-package-distributions
+ path: dist/
+
publish:
name: Upload release to PyPI
+ needs:
+ - build
runs-on: ubuntu-latest
environment:
name: pypi
@@ -18,12 +42,9 @@ jobs:
permissions:
id-token: write
steps:
- - uses: actions/checkout@v6
- - uses: actions/setup-python@v6
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- python-version: "3.14"
- - run: |
- python -m pip install --upgrade build
- python -m build
+ name: python-package-distributions
+ path: dist/
- name: Publish to PyPI
- uses: pypa/gh-action-pypi-publish@release/v1
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 566b34e50..af2a0206a 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -1,4 +1,8 @@
name: macOS
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -15,6 +19,9 @@ jobs:
runs-on: macos-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@@ -31,25 +38,39 @@ jobs:
TOXENV: no-reactor
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v6
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ run: uv tool install mitmproxy
+
- name: Run tests
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index ad2bcfce7..60a2bec21 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -1,4 +1,8 @@
name: Ubuntu
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -15,6 +19,9 @@ jobs:
runs-on: ubuntu-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@@ -92,10 +99,12 @@ jobs:
coverage: true
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v6
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
@@ -105,21 +114,32 @@ jobs:
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
- name: Install mitmproxy
- run: pipx install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ # mitmproxy has no PyPy wheels, so run it on CPython regardless of the
+ # interpreter under test.
+ run: uv tool install --python cpython mitmproxy
- name: Run tests
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index c33f96b12..884ff8e7c 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -1,4 +1,8 @@
name: Windows
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -15,6 +19,9 @@ jobs:
runs-on: windows-latest
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@@ -55,25 +62,39 @@ jobs:
TOXENV: extra-deps
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v6
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ run: uv tool install mitmproxy
+
- name: Run tests
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/codecov-action@v5
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c27348c7a..c2cb5056d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,6 +27,11 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
- rev: 0.8.9
+ rev: 0.8.10
hooks:
- id: sphinx-scrapy
+- repo: https://github.com/zizmorcore/zizmor-pre-commit
+ rev: v1.28.0
+ hooks:
+ - id: zizmor
+ args: [--no-progress, --fix]
diff --git a/docs/requirements.in b/docs/requirements.in
index 257365380..3783dd1dc 100644
--- a/docs/requirements.in
+++ b/docs/requirements.in
@@ -5,4 +5,4 @@ sphinx
sphinx-notfound-page
sphinx-rtd-theme
sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/[email protected]
+sphinx-scrapy @ git+https://github.com/scrapy/[email protected]
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 87634cea9..0f5969401 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy
diff --git a/tox.ini b/tox.ini
index 8331e2ab6..edde83356 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,8 @@
[tox]
requires =
- sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]
+ sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]
+ tox-uv
envlist =
pre-commit
pylint