Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import os
from importlib.util import find_spec
from pathlib import Path
from typing import TYPE_CHECKING, Any
from weakref import WeakSet
import pytest
from twisted.web.http import H2_ENABLED
from scrapy.core.engine import ExecutionEngine
from scrapy.utils.reactor import set_asyncio_event_loop_policy
from scrapy.utils.reactorless import install_reactor_import_hook
from tests.keys import generate_keys
from tests.mockserver.http import MockServer
from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
try:
import aiohttp
except ImportError:
aiohttp = None
if TYPE_CHECKING:
from collections.abc import Generator, Iterator
def _py_files(folder):
return (str(p) for p in Path(folder).rglob("*.py"))
collect_ignore = [
# may need extra deps
"docs/_ext",
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess
*_py_files("tests/AsyncCrawlerProcess"),
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess
*_py_files("tests/AsyncCrawlerRunner"),
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
]
base_dir = Path(__file__).parent
ignore_file_path = base_dir / "tests" / "ignores.txt"
with ignore_file_path.open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != "#":
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
"scrapy/core/downloader/handlers/http2.py",
*_py_files("scrapy/core/_http2"),
)
)
if find_spec("httpx2") is None:
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
if find_spec("pytest_codspeed") is None:
collect_ignore.append("tests/benchmarks")
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):
return
# add the full choice set so that pytest doesn't complain about invalid choices in some cases
parser.addoption(
"--reactor",
default="none",
choices=["asyncio", "default", "none"],
)
@pytest.fixture(autouse=True)
def fast_engine_heartbeat(monkeypatch: pytest.MonkeyPatch) -> None:
"""Shorten the interval at which the engine checks for work while idle.
Whenever nothing else wakes the engine up, e.g. while a component sleeps,
it stays idle until its next heartbeat, so tests that go through that wait
pay the whole interval.
"""
monkeypatch.setattr(ExecutionEngine, "_SLOT_HEARTBEAT_INTERVAL", 0.1)
_aiohttp_sessions: WeakSet[aiohttp.ClientSession] = WeakSet()
@pytest.fixture(autouse=True, scope="session")
def _track_aiohttp_sessions() -> Iterator[None]:
if aiohttp is None:
yield
return
original_init = aiohttp.ClientSession.__init__
def init(self: aiohttp.ClientSession, *args: Any, **kwargs: Any) -> None:
original_init(self, *args, **kwargs)
_aiohttp_sessions.add(self)
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(aiohttp.ClientSession, "__init__", init)
yield
@pytest.fixture(autouse=True)
def _fail_on_unclosed_aiohttp_session(
caplog: pytest.LogCaptureFixture,
) -> Iterator[None]:
"""Fail the test that leaks an aiohttp ``ClientSession`` instead of a
later, unrelated one.
A leaked session usually sits in a reference cycle with its crawler, so it
is still alive at teardown and can be checked directly. One that was
already collected during the test reported itself through
``ClientSession.__del__()``, which logs to the asyncio logger rather than
through the warnings module. Under the ``none`` reactor, any test that
downloads over HTTP or HTTPS can end up going through
:class:`~scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler`,
not only the tests that target it directly, so this runs for every test
rather than a single module.
"""
yield
unclosed = [session for session in _aiohttp_sessions if not session.closed]
_aiohttp_sessions.clear()
reported = [
record.message
for when in ("setup", "call", "teardown")
for record in caplog.get_records(when)
if record.name == "asyncio" and "Unclosed" in record.message
]
assert not unclosed
assert not reported
@pytest.fixture(scope="session")
def mockserver() -> Generator[MockServer]:
with MockServer() as mockserver:
yield mockserver
@pytest.fixture(scope="session")
def mockserver_ipv6() -> Generator[MockServer]:
"""A mockserver addressed by the IPv6 loopback.
Tests using this must skip when :func:`ipv6_loopback_available` is False.
"""
# Where IPv6 sockets also accept IPv4, binding the IPv6 wildcard keeps
# IPv4 servers from getting the same port numbers. Otherwise a client
# resolving localhost to ::1 first would reach this server instead of
# an IPv4 one.
with MockServer("::") as mockserver:
yield mockserver
@pytest.fixture(scope="session")
def _mitm_proxies() -> Generator[dict[tuple[str, str], tuple[MitmProxy, str]]]:
proxies: dict[tuple[str, str], tuple[MitmProxy, str]] = {}
try:
yield proxies
finally:
for proxy, _url in proxies.values():
proxy.stop()
@pytest.fixture # function scope because it modifies os.environ
def proxy_server(
request: pytest.FixtureRequest,
monkeypatch: pytest.MonkeyPatch,
_mitm_proxies: dict[tuple[str, str], tuple[MitmProxy, str]],
) -> str:
kind: str = request.param
# test classes can make the proxy listen on a different address, e.g. "::1"
host = getattr(request.cls, "proxy_host", "127.0.0.1")
cache_key = (kind, host)
if cache_key not in _mitm_proxies:
proxy = MitmProxy(mode="socks5" if kind == "socks5" else None, host=host)
_mitm_proxies[cache_key] = (proxy, proxy.start())
_, url = _mitm_proxies[cache_key]
if kind == "https":
url = url.replace("http://", "https://")
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
return kind
@pytest.fixture(scope="session")
def reactor_pytest(request) -> str:
return request.config.getoption("--reactor")
def pytest_configure(config):
if config.getoption("--reactor") in {"asyncio", "none"}:
# Needed on Windows to switch from proactor to selector, which supports
# add_reader/add_writer (required by the Twisted asyncio reactor, and by
# tests that register their own readers) and is what Twisted expects.
set_asyncio_event_loop_policy()
if config.getoption("--reactor") == "none":
install_reactor_import_hook()
if config.getoption("--reactor") == "default":
config.addinivalue_line(
"filterwarnings",
"ignore:Using a Twisted reactor other than:scrapy.exceptions.ScrapyDeprecationWarning",
)
# Test modules made up mostly of tests that spawn a subprocess per test (see
# tests/utils/cmdline.py). Collected in file order, they form one contiguous
# block. pytest-xdist's initial scheduling (LoadScheduling.schedule() in
# xdist/scheduler/load.py) hands each worker one contiguous slice of the
# collection, so whichever worker's slice overlaps the block ends up running
# most of it while workers with lighter slices drain them and sit idle for
# the rest of the run. Spreading these tests evenly through the whole
# collection instead means every slice gets a proportional share.
_SUBPROCESS_HEAVY_SUFFIXES = (
"tests/test_cmdline/__init__.py",
"tests/test_cmdline_crawl_with_pipeline/__init__.py",
"tests/test_command_check.py",
"tests/test_command_crawl.py",
"tests/test_command_fetch.py",
"tests/test_command_genspider.py",
"tests/test_command_parse.py",
"tests/test_command_runspider.py",
"tests/test_command_shell.py",
"tests/test_command_startproject.py",
"tests/test_command_version.py",
"tests/test_commands.py",
"tests/test_crawler_subprocess.py",
)
def _is_subprocess_heavy(item):
return item.path.as_posix().endswith(_SUBPROCESS_HEAVY_SUFFIXES)
def _interleave_evenly(items, is_heavy):
"""Merge *light* and *heavy* items so heavy ones land at roughly even
intervals throughout the result, instead of clustered together."""
light = [item for item in items if not is_heavy(item)]
heavy = [item for item in items if is_heavy(item)]
result = []
light_i = heavy_i = 0
while light_i < len(light) or heavy_i < len(heavy):
heavy_progress = heavy_i / len(heavy) if heavy else 1.0
light_progress = light_i / len(light) if light else 1.0
if heavy_i < len(heavy) and heavy_progress <= light_progress:
result.append(heavy[heavy_i])
heavy_i += 1
else:
result.append(light[light_i])
light_i += 1
return result
def pytest_collection_modifyitems(items):
items[:] = _interleave_evenly(items, _is_subprocess_heavy)
for item in items:
if item.get_closest_marker("requires_internet"):
# Requests to real websites fail every now and then in CI for
# reasons unrelated to the code under test.
item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=5))
def pytest_runtest_setup(item):
# Skip tests based on reactor markers
reactor = item.config.getoption("--reactor")
if item.get_closest_marker("requires_reactor") and reactor == "none":
pytest.skip('This test is only run when the --reactor value is not "none"')
if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
pytest.skip(
'This test is only run when the --reactor value is "asyncio" (default) or "none"'
)
if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}:
pytest.skip(
'This test is only run when the --reactor value is not "asyncio" (default) or "none"'
)
# Skip tests requiring optional dependencies
optional_deps = [
"uvloop",
"aiobotocore",
"botocore",
"boto3",
"aioboto3",
"keyring",
"psutil",
]
for module in optional_deps:
if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
pytest.skip(f"{module} is not installed")
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
pytest.skip("mitmdump is not available")
# Generate localhost certificate files, needed by some tests (but only once if xdist is used)
if "PYTEST_XDIST_WORKER" not in os.environ:
generate_keys()