Scrapy, a fast high-level web crawling & scraping framework for Python.
import asyncio
import gc
import logging
import warnings
from pathlib import Path
from typing import Any
import pytest
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from scrapy import Request, Spider, signals
from scrapy.core.downloader import Downloader
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.engine import ExecutionEngine
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.http import BodyFileResponse, HtmlResponse, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FilesPipeline
from scrapy.utils.asyncio import call_later, sleep
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
def _count_backout_logs(caplog: pytest.LogCaptureFixture) -> int:
return sum(
1
for record in caplog.records
if record.levelname == "INFO"
and str(record.msg).startswith("Pausing request processing")
)
def _backout_stats(crawler: Crawler) -> dict[str, Any]:
assert crawler.stats
return {
k: v
for k, v in crawler.stats.get_stats().items()
if k.startswith("request_backout_seconds/")
}
class OfflineSpider(Spider):
name = "offline"
start_urls = ["data:,"]
def parse(self, response):
pass
def _assert_scraper_slot_deprecation(
warning_messages: pytest.WarningsRecorder, *, ignored: bool = False
) -> None:
"""Assert that a crawl emitted exactly one Scrapy deprecation warning, the
one about SCRAPER_SLOT_MAX_ACTIVE_SIZE.
Only Scrapy deprecation warnings are counted: a crawl may emit unrelated
warnings (e.g. a ResourceWarning for a socket garbage-collected while the
recorder is active), and those must not make the assertion flaky.
Pass ``ignored=True`` when RESPONSE_MAX_ACTIVE_SIZE is set with an equal or
higher priority and therefore SCRAPER_SLOT_MAX_ACTIVE_SIZE is being
ignored."""
deprecations = [
message
for message in warning_messages
if issubclass(message.category, ScrapyDeprecationWarning)
]
assert len(deprecations) == 1
if ignored:
assert str(deprecations[0].message) == (
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated and is "
"being ignored because RESPONSE_MAX_ACTIVE_SIZE is set with an "
"equal or higher priority. Remove SCRAPER_SLOT_MAX_ACTIVE_SIZE "
"from your settings."
)
else:
assert str(deprecations[0].message) == (
"The SCRAPER_SLOT_MAX_ACTIVE_SIZE setting is deprecated, use "
"RESPONSE_MAX_ACTIVE_SIZE instead."
)
def _raise(response: Response) -> None:
raise ValueError(response.url)
def _ignore(failure: Failure) -> None:
pass
class _RaiseOnSpiderInput:
def process_spider_input(self, response: Response) -> None:
raise ValueError(response.url)
class _IgnoreResponse:
def process_response(self, request: Request, response: Response) -> Response:
raise IgnoreRequest
class _BodyFileHandler:
lazy = False
async def download_request(self, request: Request) -> BodyFileResponse:
await sleep(0.01)
body_file = request.meta["body_file"]
Path(body_file).write_bytes(b"a" * 2048)
return BodyFileResponse(request.url, body_file=body_file)
async def _crawl_without_gc(crawler: Crawler) -> None:
# The log capture of pytest keeps the tracebacks of logged errors, and with
# them the responses they refer to.
logging.disable()
gc.disable()
try:
crawl_deferred = crawler.crawl()
# A stall would otherwise hang the test.
timeout = call_later(10, crawl_deferred.cancel)
try:
await maybe_deferred_to_future(crawl_deferred)
finally:
timeout.cancel()
finally:
gc.enable()
logging.disable(logging.NOTSET)
class gt:
__hash__ = None # type: ignore[assignment]
def __init__(self, value: float):
self.value = value
def __eq__(self, other: object) -> bool:
return isinstance(other, (int, float)) and other > self.value
def __repr__(self) -> str:
return f">{self.value}"
class TestResponseMaxActiveSize:
@coroutine_test
async def test_default(self):
"""A crawl without custom settings has its effective response max
active size set to 5 000 000, and triggers no deprecation warning."""
crawler = get_crawler(OfflineSpider)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 5_000_000
@coroutine_test
async def test_custom(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE to a custom value changes the
effective response max active size."""
crawler = get_crawler(
OfflineSpider, settings_dict={"RESPONSE_MAX_ACTIVE_SIZE": 0}
)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 0
@coroutine_test
async def test_deprecated_default(self):
"""Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE triggers a deprecation warning,
even if it is the default value."""
crawler = get_crawler(
OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 5_000_000}
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 5_000_000
_assert_scraper_slot_deprecation(warning_messages)
@coroutine_test
async def test_deprecated_custom(self):
"""Setting SCRAPER_SLOT_MAX_ACTIVE_SIZE to a custom value triggers a
deprecation warning, and changes the effective response max active
size."""
crawler = get_crawler(
OfflineSpider, settings_dict={"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 0}
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 0
_assert_scraper_slot_deprecation(warning_messages)
@coroutine_test
async def test_both(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to
different values with the same setting priority triggers a deprecation
warning about SCRAPER_SLOT_MAX_ACTIVE_SIZE being ignored, and makes
the value of RESPONSE_MAX_ACTIVE_SIZE the effective response max active
size."""
crawler = get_crawler(
OfflineSpider,
settings_dict={
"RESPONSE_MAX_ACTIVE_SIZE": 1,
"SCRAPER_SLOT_MAX_ACTIVE_SIZE": 2,
},
)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 1
_assert_scraper_slot_deprecation(warning_messages, ignored=True)
@coroutine_test
async def test_both_deprecated_priority(self):
"""Setting RESPONSE_MAX_ACTIVE_SIZE and SCRAPER_SLOT_MAX_ACTIVE_SIZE to
different values and SCRAPER_SLOT_MAX_ACTIVE_SIZE with a higher
priority triggers a deprecation warning about
SCRAPER_SLOT_MAX_ACTIVE_SIZE but also makes the value of
SCRAPER_SLOT_MAX_ACTIVE_SIZE the effective response max active size."""
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
@classmethod
def update_settings(cls, settings):
settings.set("RESPONSE_MAX_ACTIVE_SIZE", 1, priority=100)
settings.set("SCRAPER_SLOT_MAX_ACTIVE_SIZE", 2, priority=101)
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
with pytest.warns(ScrapyDeprecationWarning) as warning_messages:
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._max_active_size == 2
_assert_scraper_slot_deprecation(warning_messages)
class TestResponseRoughSize:
@pytest.fixture(autouse=True)
def use_caplog(self, caplog):
self.caplog = caplog
@pytest.mark.parametrize(
("settings_dict", "expected"),
[
# A quarter of RESPONSE_MAX_ACTIVE_SIZE split among
# CONCURRENT_REQUESTS requests.
({}, 78125),
({"CONCURRENT_REQUESTS": 100}, 12500),
({"RESPONSE_MAX_ACTIVE_SIZE": 8_000_000}, 125_000),
# Unlimited concurrency leaves no number of requests to split it
# among.
({"CONCURRENT_REQUESTS": 0}, 0),
({"RESPONSE_MAX_ACTIVE_SIZE": 0}, 0),
({"RESPONSE_ROUGH_SIZE": 1}, 1),
({"RESPONSE_ROUGH_SIZE": 0}, 0),
],
)
@coroutine_test
async def test_value(self, settings_dict, expected):
crawler = get_crawler(OfflineSpider, settings_dict=settings_dict)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
assert crawler.engine.downloader.middleware._response_rough_size == expected
@coroutine_test
async def test_response_replaces_rough_size(self):
"""The rough size of a request stops counting as soon as the size of its
response is known."""
sizes: list[tuple[int, int]] = []
class DownloaderMiddleware:
def __init__(self, crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_response(self, request, response):
middleware = self.crawler.engine.downloader.middleware
sizes.append(
(middleware._rough_active_size, middleware._response_active_size)
)
return response
class TestSpider(Spider):
name = "test"
start_urls = ["data:,a"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {DownloaderMiddleware: 0},
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
assert sizes == [(0, 1)]
assert crawler.engine
assert crawler.engine.downloader.middleware._rough_sizes == {}
@pytest.mark.only_asyncio
@coroutine_test
async def test_same_request_downloaded_twice(self):
"""The rough size of a request object being downloaded twice at the same
time is counted once, so that it is gone once both downloads finish."""
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {"RESPONSE_ROUGH_SIZE": 1024}
async def parse(self, response):
assert self.crawler.engine
request = Request("data:,a")
# Only one of the two downloads succeeds, the other one fails
# because the request object is already being downloaded.
await asyncio.gather(
self.crawler.engine.download_async(request),
self.crawler.engine.download_async(request),
return_exceptions=True,
)
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.engine
middleware = crawler.engine.downloader.middleware
assert middleware._rough_sizes == {}
assert middleware._rough_active_size == 0
@coroutine_test
async def test_rough_size_triggers_backout(self):
"""Rough sizes of requests being downloaded count toward the limit, even
if their responses turn out to be empty."""
class SlowDown:
"""Keeps requests in flight long enough for the engine to check for
backout while their rough size is being counted."""
async def process_request(self, request):
await sleep(0.01)
class TestSpider(Spider):
name = "test"
start_urls = ["data:,", "data:,"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {SlowDown: 0},
"RESPONSE_MAX_ACTIVE_SIZE": 512,
"RESPONSE_ROUGH_SIZE": 1024,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@coroutine_test
async def test_body_file(self, tmp_path):
"""Requests and responses whose body is written to a file do not count
toward the limit."""
class TestSpider(Spider):
name = "test"
custom_settings = {
"DOWNLOAD_HANDLERS": {"https": _BodyFileHandler},
"RESPONSE_MAX_ACTIVE_SIZE": 512,
"RESPONSE_ROUGH_SIZE": 1024,
}
async def start(self):
for n in range(2):
yield Request(
f"https://example.com/{n}",
meta={"body_file": tmp_path / str(n)},
)
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats
assert crawler.stats.get_value("response_received_count") == 2
assert _count_backout_logs(self.caplog) == 0
assert _backout_stats(crawler) == {}
@pytest.mark.only_asyncio
@coroutine_test
async def test_body_file_download_async(self, tmp_path):
"""download_async() requests whose response body is written to a file
are not held back by the limit."""
active = 0
max_active = 0
class Handler(_BodyFileHandler):
async def download_request(self, request: Request) -> BodyFileResponse:
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
try:
return await super().download_request(request)
finally:
active -= 1
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"DOWNLOAD_HANDLERS": {"https": Handler},
"RESPONSE_MAX_ACTIVE_SIZE": 512,
"RESPONSE_ROUGH_SIZE": 1024,
}
async def parse(self, response):
assert self.crawler.engine
await asyncio.gather(
*(
self.crawler.engine.download_async(
Request(
f"https://example.com/{n}",
meta={"body_file": tmp_path / str(n)},
)
)
for n in range(2)
)
)
yield {}
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == 1
assert max_active == 2
class TestRequestBackout:
@pytest.fixture(autouse=True)
def use_caplog(self, caplog):
self.caplog = caplog
@coroutine_test
async def test_none(self):
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 0
assert _backout_stats(crawler) == {}
@coroutine_test
async def test_concurrency(self):
class SlowDown:
"""Downloader middleware that returns a non-instant deferred from
process_request, to force need_backout calls to happen at that
point.
The delay is deliberately non-zero so that the concurrency backout
state lasts a measurable amount of wall-clock time, which keeps the
request_backout_seconds/concurrency stat reliably above 0."""
def process_request(self, request):
from twisted.internet import reactor
d: Deferred[None] = Deferred()
reactor.callLater(0.01, d.callback, None)
return d
class TestSpider(Spider):
name = "test"
# Several start URLs so that, with CONCURRENT_REQUESTS=1, the engine
# reliably attempts to schedule a second request while the first one
# is still active, which is what triggers the concurrency backout.
start_urls = ["data:,"] * 5
custom_settings = {
"CONCURRENT_REQUESTS": 1,
"DOWNLOADER_MIDDLEWARES": {SlowDown: 0},
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 0
expected_stats = {
"request_backout_seconds/concurrency": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@coroutine_test
async def test_response_size(self):
class TestSpider(Spider):
name = "test"
start_urls = ["data:,a"]
custom_settings = {
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@pytest.mark.parametrize(
("status", "callback", "errback", "settings"),
[
(200, lambda response: response.css("a"), _ignore, {}),
(200, _raise, _ignore, {}),
(404, None, _ignore, {}),
(404, None, None, {}),
(404, None, lambda failure: failure, {}),
(
200,
None,
None,
{"SPIDER_MIDDLEWARES": {_RaiseOnSpiderInput: 0}},
),
(
200,
None,
None,
{"DOWNLOADER_MIDDLEWARES": {_IgnoreResponse: 1}},
),
],
ids=[
"selector",
"callback-exception",
"errback",
"no-errback",
"errback-failure",
"spider-middleware-exception",
"downloader-middleware-ignore",
],
)
@coroutine_test
async def test_response_max_active_size_no_cycles(
self, monkeypatch, status, callback, errback, settings
):
"""Scrapy frees responses through reference counting alone, so a crawl
does not stall on RESPONSE_MAX_ACTIVE_SIZE without garbage
collection."""
# Keep the bypass, which collects garbage first, from lifting a stall.
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 60)
class ResponseMiddleware:
def process_request(self, request):
return HtmlResponse(request.url, status=status, body=b"<a></a>")
class TestSpider(Spider):
name = "test"
custom_settings = {
**settings,
"DOWNLOADER_MIDDLEWARES": {
ResponseMiddleware: 0,
**settings.get("DOWNLOADER_MIDDLEWARES", {}),
},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
async def start(self):
for n in range(3):
yield Request(
f"https://example.com/{n}", callback=callback, errback=errback
)
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
await _crawl_without_gc(crawler)
assert crawler.stats
assert crawler.stats.get_value("downloader/response_count") == 3
@coroutine_test
async def test_response_max_active_size_no_cycles_media(
self, monkeypatch, tmp_path
):
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 60)
class ResponseMiddleware:
def process_request(self, request):
status = 404 if request.url.endswith("/file") else 200
return Response(request.url, status=status, body=b"a")
class TestSpider(Spider):
name = "test"
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {ResponseMiddleware: 0},
"FILES_STORE": str(tmp_path),
"ITEM_PIPELINES": {FilesPipeline: 1},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
async def start(self):
for n in range(3):
yield Request(f"https://example.com/{n}")
def parse(self, response):
yield {"file_urls": [f"{response.url}/file"]}
crawler = get_crawler(TestSpider)
await _crawl_without_gc(crawler)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == 3
@coroutine_test
async def test_response_size_process_request(self):
class DownloaderMiddleware:
def process_request(self, request):
return Response("https://example.com", body=b"a")
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {DownloaderMiddleware: 0},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@coroutine_test
async def test_request_from_process_request(self):
"""A request returned from process_request does not count toward the
limit, even though requests also have a body."""
class DownloaderMiddleware:
def __init__(self):
self.replaced = False
def process_request(self, request):
if self.replaced:
return None
self.replaced = True
return Request("data:,b", body=b"a" * 2000)
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {DownloaderMiddleware: 0},
"RESPONSE_MAX_ACTIVE_SIZE": 1000,
"RESPONSE_ROUGH_SIZE": 0,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 0
assert _backout_stats(crawler) == {}
@coroutine_test
async def test_response_size_process_response(self):
class DownloaderMiddleware:
def process_response(self, request, response):
return Response("https://example.com", body=b"a")
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {DownloaderMiddleware: 0},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@coroutine_test
async def test_response_size_process_exception(self):
class DownloaderMiddleware1:
def process_exception(self, request, exception):
return Response("https://example.com", body=b"a")
class DownloaderMiddleware2:
def process_request(self, request):
raise ValueError
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"DOWNLOADER_MIDDLEWARES": {
DownloaderMiddleware1: 0,
DownloaderMiddleware2: 1,
},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
def parse(self, response):
pass
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@coroutine_test
async def test_response_size_download(self):
"""Ensure that responses from engine.download calls are also taken into
account for the RESPONSE_MAX_ACTIVE_SIZE setting."""
class SlowDown:
"""Item pipeline that returns a non-instant deferred, to force
need_backout calls to happen at that point."""
def process_item(self, item):
from twisted.internet import reactor
d: Deferred[dict[Any, Any]] = Deferred()
reactor.callLater(0, d.callback, {})
return d
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"ITEM_PIPELINES": {SlowDown: 0},
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
async def parse(self, response):
assert self.crawler.engine
response = await self.crawler.engine.download_async(Request("data:,a"))
yield {"response": response}
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert _count_backout_logs(self.caplog) == 1
expected_stats = {
"request_backout_seconds/response_max_active_size": gt(0),
"request_backout_seconds/total": gt(0),
}
assert _backout_stats(crawler) == expected_stats
@pytest.mark.parametrize("shared_slot", [True, False])
@coroutine_test
async def test_response_max_active_size_media_downloads(self, shared_slot):
"""Responses of media pipeline requests waiting for a slow upload hold
back the next media downloads.
"""
max_active_size = 3000
uploads = 0
peak_uploads = 0
class SlowStore(_NoStatStore):
async def persist_file(self, path, buf, info, meta=None, headers=None):
nonlocal uploads, peak_uploads
uploads += 1
peak_uploads = max(peak_uploads, uploads)
await sleep(0.05)
uploads -= 1
class TestSpider(_MediaSpider):
file_count = 20
file_size = 1000
custom_settings = {
"FILES_STORE": "test://",
"ITEM_PIPELINES": {
_SharedSlotFilesPipeline
if shared_slot
else _SlotPerFileFilesPipeline: 1
},
"MEDIA_STORAGES": {"test": SlowStore},
"RESPONSE_MAX_ACTIVE_SIZE": max_active_size,
"RESPONSE_ROUGH_SIZE": file_size,
}
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats
assert crawler.stats.get_value("file_count") == 20
assert peak_uploads == max_active_size // TestSpider.file_size
assert crawler.stats.get_value("response_max_active_size/bypass_count") is None
@coroutine_test
async def test_response_max_active_size_scheduled_response_with_files(self):
"""A scheduled response above the limit does not hold back the media
downloads that it waits for.
"""
class TestSpider(_MediaSpider):
start_urls = [f"data:,{'a' * 2000}"]
custom_settings = {
"FILES_STORE": "test://",
"ITEM_PIPELINES": {_SharedSlotFilesPipeline: 1},
"MEDIA_STORAGES": {"test": _NoStatStore},
"RESPONSE_MAX_ACTIVE_SIZE": 1000,
}
file_count = 3
file_size = 10
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats
assert crawler.stats.get_value("file_count") == 3
assert crawler.stats.get_value("response_max_active_size/bypass_count") is None
@coroutine_test
async def test_response_max_active_size_scheduled_request_behind_held_back(
self, monkeypatch
):
"""Scheduled requests queued in a slot behind a held-back
download_async() request are downloaded first, and the bypass lets it
through even if it is due while the slot waits for its download
delay. Once it is through, scheduled requests are not held back.
"""
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 0.01)
downloads: list[int | str] = []
scheduled_reached: Deferred[None] = Deferred()
scheduled_downloaded: Deferred[None] = Deferred()
class Handler(DataURIDownloadHandler):
async def download_request(self, request: Request) -> Response:
# Keeps the slot busy until the scheduled requests are queued
# behind the held-back one.
if request.meta.get("wait_for_scheduled"):
await maybe_deferred_to_future(scheduled_reached)
return await super().download_request(request)
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"CONCURRENT_REQUESTS_PER_DOMAIN": 1,
"DOWNLOAD_DELAY": 0.2,
"DOWNLOAD_DELAY_JITTER": 0,
"DOWNLOAD_HANDLERS": {"data": Handler},
"RESPONSE_MAX_ACTIVE_SIZE": 1000,
"RESPONSE_ROUGH_SIZE": 1,
}
scheduled_count = 0
def request_reached_downloader(self, request: Request) -> None:
if request.url[-1].isdigit():
self.scheduled_count += 1
if self.scheduled_count == 4:
scheduled_reached.callback(None)
def response_downloaded(self, response: Response) -> None:
if response.url == "data:,a":
downloads.append("a")
elif response.url[-1].isdigit():
downloads.append(int(response.text))
if len(downloads) == 5:
scheduled_downloaded.callback(None)
async def parse(self, response):
assert self.crawler.engine
self.crawler.signals.connect(
self.request_reached_downloader,
signals.request_reached_downloader,
)
self.crawler.signals.connect(
self.response_downloaded, signals.response_downloaded
)
download_async = self.crawler.engine.download_async
kept_dfd = deferred_from_coro(
download_async(
Request(
f"data:,{'a' * 2000}", meta={"wait_for_scheduled": True}
)
)
)
held_back_dfd = deferred_from_coro(download_async(Request("data:,a")))
for n in range(4):
yield Request(f"data:,{n}", callback=self.parse_b)
kept = await maybe_deferred_to_future(kept_dfd)
await maybe_deferred_to_future(held_back_dfd)
await maybe_deferred_to_future(scheduled_downloaded)
assert kept
yield {}
def parse_b(self, response):
pass
crawler = get_crawler(TestSpider)
crawl_deferred = crawler.crawl()
# Scheduled requests held back after the bypass would otherwise hang
# the test.
timeout = call_later(10, crawl_deferred.cancel)
try:
await maybe_deferred_to_future(crawl_deferred)
finally:
timeout.cancel()
assert crawler.stats
assert downloads == [3, "a", 2, 1, 0]
assert crawler.stats.get_value("response_max_active_size/bypass_count") == 1
@pytest.mark.parametrize(
("kept_count", "kept_size", "examples", "more"),
[(1, 2000, 1, ""), (4, 300, 3, " (+1 more)")],
)
@coroutine_test
async def test_response_max_active_size_bypass(
self, monkeypatch, kept_count, kept_size, examples, more
):
"""Code that keeps a response to a download_async() request above the
limit while it awaits another one slows down instead of stalling.
"""
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 0.01)
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"]
custom_settings = {
"RESPONSE_MAX_ACTIVE_SIZE": 1000,
}
async def parse(self, response):
assert self.crawler.engine
kept = [
await self.crawler.engine.download_async(
Request(f"data:,{n:0{kept_size}}")
)
for n in range(kept_count)
]
for n in range(2):
await self.crawler.engine.download_async(Request(f"data:,{n}"))
assert kept
yield {}
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
await maybe_deferred_to_future(crawler.crawl())
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == 1
assert crawler.stats.get_value("response_max_active_size/bypass_count") == 2
messages = [
record.getMessage()
for record in self.caplog.records
if record.getMessage().startswith("Letting")
]
assert len(messages) == 1
assert "<GET data:,0>" in messages[0]
assert messages[0].count("<200 data:,") == examples
assert f">{more}, have used" in messages[0]
@coroutine_test
async def test_response_max_active_size_scheduled_bypass(self, monkeypatch):
"""Code that keeps responses to scheduled requests above the limit
slows down instead of stalling.
"""
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 0.01)
monkeypatch.setattr(ExecutionEngine, "_SLOT_HEARTBEAT_INTERVAL", 0.01)
class TestSpider(Spider):
name = "test"
start_urls = [f"data:,{n}" for n in range(3)]
custom_settings = {
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
kept: list[Response] = []
def parse(self, response):
self.kept.append(response)
crawler = get_crawler(TestSpider)
self.caplog.clear()
with self.caplog.at_level("INFO"):
crawl_deferred = crawler.crawl()
# A stall would otherwise hang the test.
timeout = call_later(10, crawl_deferred.cancel)
try:
await maybe_deferred_to_future(crawl_deferred)
finally:
timeout.cancel()
assert crawler.stats
assert crawler.stats.get_value("response_received_count") == 3
assert crawler.stats.get_value("response_max_active_size/bypass_count") == 2
records = [
record
for record in self.caplog.records
if record.getMessage().startswith("Letting")
]
assert len(records) == 1
assert records[0].levelno == logging.WARNING
assert "<200 data:," in records[0].getMessage()
@pytest.mark.parametrize("awaited", [False, True])
@coroutine_test
async def test_response_max_active_size_collects_cycles(self, monkeypatch, awaited):
"""Responses in a reference cycle stop counting toward the limit once
it holds back requests, even with automatic garbage collection
disabled.
"""
monkeypatch.setattr(Downloader, "_SIZE_BYPASS_DELAY", 0.01)
monkeypatch.setattr(ExecutionEngine, "_SLOT_HEARTBEAT_INTERVAL", 0.01)
def keep_in_cycle(response: Response) -> None:
cycle: list[Any] = [response]
cycle.append(cycle)
class TestSpider(Spider):
name = "test"
start_urls = ["data:,"] if awaited else [f"data:,{n}" for n in range(3)]
custom_settings = {
"RESPONSE_MAX_ACTIVE_SIZE": 1,
}
async def parse(self, response):
if not awaited:
keep_in_cycle(response)
yield {}
return
assert self.crawler.engine
for n in range(3):
keep_in_cycle(
await self.crawler.engine.download_async(Request(f"data:,{n}"))
)
yield {}
crawler = get_crawler(TestSpider)
await _crawl_without_gc(crawler)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == 3
assert crawler.stats.get_value("response_max_active_size/bypass_count") is None
class _NoStatStore:
checksum_algorithm = "md5"
def __init__(self, uri: str):
pass
def stat_file(self, path, info):
return {}
def persist_file(self, path, buf, info, meta=None, headers=None):
pass
class _SharedSlotFilesPipeline(FilesPipeline):
pass
class _SlotPerFileFilesPipeline(FilesPipeline):
def get_media_requests(self, item, info):
return [
Request(url, meta={"download_slot": url}, callback=NO_CALLBACK)
for url in item["file_urls"]
]
class _MediaSpider(Spider):
name = "test"
start_urls = ["data:,"]
file_count: int
file_size: int
def parse(self, response):
yield {
"file_urls": [
f"data:,{n:0{self.file_size}}" for n in range(self.file_count)
]
}