Scrapy, a fast high-level web crawling & scraping framework for Python.
# Tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
requires =
sphinx-scrapy[tox]==0.15.0
tox-uv
envlist =
pre-commit
pylint
mypy
mypy-tests
twinecheck
docs
docs-tests
docs-links
docs-coverage
min
min-extra-deps
min-default-reactor
min-no-reactor
min-botocore
py{310,311,312,313,314,315}
extra-deps
default-reactor
no-reactor
no-reactor-extra-deps
botocore
benchmark
minversion = 1.7.0
[test-requirements]
deps =
attrs
coverage >= 7.10.6
pexpect >= 4.8.0
psutil
pyftpdlib >= 2.0.1
pygments
pytest
pytest-cov >= 7.0.0
pytest-rerunfailures
pytest-timeout
pytest-xdist
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
pytest-twisted >= 1.14.3
[testenv]
deps =
{[test-requirements]deps}
pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502
passenv =
PYTHONTRACEMALLOC
PYTEST_ADDOPTS
MITMDUMP
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN
#allow tox virtualenv to upgrade pip/wheel/setuptools
download = true
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
[testenv:mypy]
basepython = python3.11
deps =
mypy==2.4.1
typing-extensions==4.16.0
Pillow==13.0.0
keyring==22.3.0
Protego==0.7.0
Twisted==26.10.0
aiodns==4.0.4
aioftp==0.28.3
aiohttp==3.14.5
attrs==26.1.0
backports.zstd==1.8.0
boto3-stubs[s3]==1.43.125
botocore-stubs==1.43.125
colorlog==6.12.0
frostwork==0.2.2
h2==4.4.1
httpx2==2.16.0
itemadapter==0.14.0
platformdirs==4.14.5
ptpython==3.0.32
ipython==9.18.1
pyOpenSSL==26.4.0
pytest==9.2.0
python-socks[asyncio]==3.2.1
socksio==1.0.0
types-Pygments==2.21.0.20260819
types-defusedxml==0.7.0.20260504
types-lxml==2026.2.16
types-pexpect==4.9.0.20260518
uvloop==0.23.0
w3lib==2.6.0
commands =
mypy {posargs:scrapy tests}
[testenv:mypy-tests]
basepython = python3.11
deps =
{[test-requirements]deps}
{[testenv:mypy]deps}
pytest-mypy-testing==0.2.0
commands =
pytest {posargs:tests_typing}
[testenv:pre-commit]
basepython = python3
deps =
pre-commit
commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
# Some checks are Python-version-dependent, so pin the version used in CI.
# 3.15 with jobs=0 needs dill > 0.4.1 which isn't released yet
basepython = python3.14
deps =
{[testenv:extra-deps]deps}
pylint==4.1.3
pylint-per-file-ignores==4.0.0
commands =
pylint {posargs:conftest.py docs extras scrapy tests}
[testenv:twinecheck]
basepython = python3
deps =
twine==7.0.0
build==1.6.1
commands =
python -m build --sdist
twine check dist/*
[min]
basepython = python3.11
deps =
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
pytest==8.4.0
Protego==0.2.0
Twisted==23.10.0
aiodns==3.3.0
aioftp==0.26.1
aiohttp==3.13.3
backports.zstd==1.3.0
brotli==1.2.0
cryptography==41.0.5
cssselect==1.2.0
formerly==0.1.0
frostwork==0.2.2
itemadapter==0.1.0
itemloaders==1.0.5
lxml==4.9.4
parsel==1.8.1
platformdirs==2.0.0
# aiodns < 4 needs pycares < 5
pycares==4.11.0
pyOpenSSL==24.3.0
queuelib==1.8.0
service_identity==24.2.0
typing-extensions==4.13.0
w3lib==2.6.0
zope.interface==5.1.0
{[test-requirements]deps}
setenv =
_SCRAPY_MIN=true
# Pinned old Python and library versions trigger warnings that we cannot
# fix, so these envs do not turn warnings into errors.
PYTEST_ADDOPTS=-W default {env:PYTEST_ADDOPTS:}
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}
[testenv:min]
basepython = {[min]basepython}
deps =
{[min]deps}
PyDispatcher==2.0.5
setenv =
{[min]setenv}
commands = {[min]commands}
[testenv:extra-deps]
basepython = python3
deps =
{[testenv]deps}
Pillow
keyring
Twisted[http2]
python-socks[asyncio]
siosocks # SOCKS server for tests
aioboto3
aiobotocore
boto3
bpython # optional for shell wrapper tests
colorlog
google-cloud-storage
httpx2[http2,socks]
ipython
psutil
ptpython # optional for shell wrapper tests
robotexclusionrulesparser
uvloop; platform_system != "Windows"
[testenv:min-extra-deps]
basepython = {[min]basepython}
deps =
{[min]deps}
Pillow==9.2.0
keyring==21.7.0
Twisted[http2]==23.10.0
h2==3.2.0
python-socks[asyncio]==2.6.0
siosocks==0.2.0 # SOCKS server for tests
boto3==1.35.2
bpython==0.7.1
colorlog==6.6.0
google-cloud-storage==1.30.0
httpx2[http2,socks]==2.3.0
ipython==8.15.0
psutil==5.9.0
ptpython==3.0.23
robotexclusionrulesparser==1.6.2
uvloop==0.17.0; platform_system != "Windows"
setenv =
{[min]setenv}
SCRAPY_COVERAGE_PATCH=_exit # redundant with "extra-deps"'s subprocess coverage of scrapy/commands/*
commands = {[min]commands}
[testenv:vcs-deps]
basepython = python3
deps =
{[testenv:extra-deps]deps}
uv
# Dependencies cap each other at their latest release, so their development
# branches usually cannot be resolved together: pyOpenSSL, for one, requires a
# cryptography older than the one cryptography itself is heading towards.
# --no-deps skips resolution entirely, replacing only these distributions and
# leaving the rest of the environment as the install above resolved it.
#
# Pillow and uvloop build from source, and need the libjpeg headers and
# autotools respectively. robotexclusionrulesparser has no public repository,
# and PyDispatcher has seen no commit since 2023-10-23, so they stay at their
# latest release.
commands_pre =
uv pip install --python {envpython} --no-deps --reinstall \
git+https://github.com/twisted/twisted \
git+https://github.com/python-pillow/Pillow \
git+https://github.com/MagicStack/uvloop \
git+https://github.com/aio-libs/aiodns \
git+https://github.com/aio-libs/aiohttp \
git+https://github.com/aio-libs/aioftp \
git+https://github.com/pyca/cryptography \
git+https://github.com/scrapy/cssselect \
git+https://github.com/tiran/defusedxml \
git+https://github.com/scrapy/itemadapter \
git+https://github.com/scrapy/itemloaders \
git+https://github.com/lxml/lxml \
git+https://github.com/pypa/packaging \
git+https://github.com/scrapy/parsel \
git+https://github.com/scrapy/protego \
git+https://github.com/pyca/pyopenssl \
git+https://github.com/scrapy/queuelib \
git+https://github.com/pyca/service-identity \
git+https://github.com/john-kurkowski/tldextract \
git+https://github.com/scrapy/w3lib \
git+https://github.com/zopefoundation/zope.interface \
git+https://github.com/boto/boto3 \
git+https://github.com/bpython/bpython \
git+https://github.com/google/brotli \
git+https://github.com/googleapis/python-storage \
git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \
git+https://github.com/ipython/ipython \
git+https://github.com/prompt-toolkit/ptpython
[testenv:default-reactor]
commands =
{[testenv]commands} --reactor=default
setenv =
SCRAPY_COVERAGE_PATCH=_exit # redundant with "py"'s subprocess coverage of scrapy/commands/*
[testenv:min-default-reactor]
basepython = {[min]basepython}
deps = {[testenv:min]deps}
commands = {[min]commands} --reactor=default
setenv =
{[min]setenv}
SCRAPY_COVERAGE_PATCH=_exit # redundant with "py"'s subprocess coverage of scrapy/commands/*
[testenv:no-reactor]
deps =
{[testenv]deps}
pytest-asyncio
commands =
{[testenv]commands} -p no:twisted --reactor=none
setenv =
SCRAPY_COVERAGE_PATCH=_exit # redundant with "py"'s subprocess coverage of scrapy/commands/*
[testenv:no-reactor-extra-deps]
deps =
{[testenv:extra-deps]deps}
pytest-asyncio
commands =
{[testenv]commands} -p no:twisted --reactor=none
setenv =
SCRAPY_COVERAGE_PATCH=_exit # redundant with "extra-deps"'s subprocess coverage of scrapy/commands/*
[testenv:min-no-reactor]
basepython = {[min]basepython}
deps =
{[testenv:min]deps}
pytest-asyncio
commands = {[min]commands} -p no:twisted --reactor=none
setenv =
{[min]setenv}
SCRAPY_COVERAGE_PATCH=_exit # redundant with "py"'s subprocess coverage of scrapy/commands/*
[testenv:docs-tests]
changedir = docs
deps =
{[test-requirements]deps}
form2request
sphinx-scrapy==0.15.0
dependency_groups = docs
commands =
pytest
[testenv:docs-coverage]
changedir = docs
deps =
sphinx-scrapy==0.15.0
dependency_groups = docs
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
[testenv:docs-links]
changedir = docs
deps =
sphinx-scrapy==0.15.0
dependency_groups = docs
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
# Run S3 tests with botocore installed but without boto3.
[testenv:botocore]
deps =
{[testenv]deps}
aiobotocore
botocore>=1.13.45
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m "requires_botocore or requires_aiobotocore"
[testenv:min-botocore]
basepython = {[min]basepython}
deps =
{[min]deps}
botocore==1.13.45
# botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3
requests<2.33.0
setenv =
{[min]setenv}
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
# CPU benchmarks, tracked on CodSpeed.
#
# pytest-twisted is left out on purpose: benchmarked code must be callable
# synchronously, so tests/benchmarks drives the event loop itself.
[testenv:benchmark]
basepython = python3.15
deps =
pytest >= 8.4.1
pytest-codspeed
passenv =
*codspeed*
*ci*
commands =
# No --codspeed-mode: CodSpeedHQ/action re-invokes this once per
# configured mode, selecting it through CODSPEED_RUNNER_MODE (passed
# through by the *codspeed* passenv above); a hardcoded mode here would
# override that on every run. Without it (e.g. running this tox env
# locally), pytest-codspeed defaults to walltime.
pytest {posargs:tests/benchmarks} --codspeed