Scrapy, a fast high-level web crawling & scraping framework for Python.
Address recursion and media ignore-request reporting issues (#7673)
Adrian25e6884e2f24
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 1033e874f..efac5f71c 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -470,16 +470,17 @@ class ExecutionEngine:"""if self.spider is None:raise RuntimeError(f"No open spider to crawl: {request}")- try:- response_or_request = await maybe_deferred_to_future(- self._download(request)- )- finally:- assert self._slot is not None- self._slot.remove_request(request)- if isinstance(response_or_request, Request):- return await self.download_async(response_or_request)- return response_or_request+ while True:+ try:+ response_or_request = await maybe_deferred_to_future(+ self._download(request)+ )+ finally:+ assert self._slot is not None+ self._slot.remove_request(request)+ if not isinstance(response_or_request, Request):+ return response_or_request+ request = response_or_request@inlineCallbacksdef _download(diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.pyindex e03dc040e..db85b62a1 100644--- a/scrapy/downloadermiddlewares/offsite.py+++ b/scrapy/downloadermiddlewares/offsite.py@@ -61,7 +61,7 @@ class OffsiteMiddleware:)self.stats.inc_value("offsite/domains")self.stats.inc_value("offsite/filtered")- raise IgnoreRequest+ raise IgnoreRequest(f"Filtered offsite request to {domain!r}")def should_follow(self, request: Request, spider: Spider) -> bool:regex = self.host_regexdiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 70dc36e35..8e8082332 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -27,7 +27,15 @@ from twisted.internet.defer import Deferred, maybeDeferredfrom scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACK-from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline+from scrapy.pipelines.media import (+ FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility+)+from scrapy.pipelines.media import (+ FileInfo,+ FileInfoOrError,+ MediaPipeline,+ _MediaRequestFiltered,+)from scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.datatypes import CaseInsensitiveDict@@ -75,10 +83,6 @@ def _md5sum(file: IO[bytes]) -> str:return m.hexdigest()-class FileException(Exception):- """General media error exception"""--class StatInfo(TypedDict, total=False):checksum: strlast_modified: float@@ -597,20 +601,20 @@ class FilesPipeline(MediaPipeline):def media_failed(self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo) -> NoReturn:- if not isinstance(failure.value, IgnoreRequest):- referer = referer_str(request)- logger.warning(- "File (unknown-error): Error downloading %(medianame)s from "- "%(request)s referred in <%(referer)s>: %(exception)s",- {- "medianame": self.MEDIA_NAME,- "request": request,- "referer": referer,- "exception": failure.value,- },+ referer = referer_str(request)+ if isinstance(failure.value, IgnoreRequest):+ logger.debug(+ f"File (filtered): Not downloading {self.MEDIA_NAME} from "+ f"{request} referred in <{referer}>: {failure.value}",extra={"spider": info.spider},)+ raise _MediaRequestFiltered(str(failure.value)) from failure.value+ logger.warning(+ f"File (unknown-error): Error downloading {self.MEDIA_NAME} from "+ f"{request} referred in <{referer}>: {failure.value}",+ extra={"spider": info.spider},+ )raise FileExceptionasync def media_downloaded(diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 5b5d2dcb2..764f82a78 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -55,6 +55,25 @@ FileInfoOrError: TypeAlias = (logger = logging.getLogger(__name__)+class FileException(Exception):+ """General media error exception"""+++class _MediaRequestFiltered(FileException):+ """Raised internally by media pipelines when a media request is filtered+ out (e.g. as an offsite request) instead of being downloaded.++ It is a subclass of :exc:`FileException` for backward compatibility, but+ unlike an actual download error it is logged at the ``DEBUG`` level and+ without a traceback, since filtering a request is expected behavior rather+ than an error.+ """+++def _media_request_filtered(failure: Failure) -> bool:+ return isinstance(failure.value, _MediaRequestFiltered)++class MediaPipeline(ABC):LOG_FAILED_RESULTS: bool = True@@ -193,7 +212,8 @@ class MediaPipeline(ABC):result = await self._check_media_to_download(request, info, item=item)except Exception:result = Failure()- logger.exception(result)+ if not _media_request_filtered(result):+ logger.exception(result)self._cache_result_and_execute_waiters(result, fp, info)return await maybe_deferred_to_future(wad) # it must return wad at last@@ -304,6 +324,8 @@ class MediaPipeline(ABC):for ok, value in results:if not ok:assert isinstance(value, Failure)+ if _media_request_filtered(value):+ continuelogger.error("%(class)s found errors processing %(item)s",{"class": self.__class__.__name__, "item": item},diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex c0b8dc4dd..78efb0191 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -1,3 +1,5 @@+import re+import pytestfrom scrapy import Request, Spider@@ -231,3 +233,15 @@ def test_repeated_offsite_domain():mw.process_request(req2)assert crawler.stats.get_value("offsite/domains") == 1 # not incremented againassert crawler.stats.get_value("offsite/filtered") == 2+++def test_ignore_request_reason():+ crawler = get_crawler(Spider)+ crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])+ mw = OffsiteMiddleware.from_crawler(crawler)+ mw.spider_opened(crawler.spider)+ request = Request("http://other.org/1")+ with pytest.raises(+ IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")+ ):+ mw.process_request(request)diff --git a/tests/test_engine_download.py b/tests/test_engine_download.pyindex f15bfd5e2..962808d96 100644--- a/tests/test_engine_download.py+++ b/tests/test_engine_download.py@@ -1,5 +1,6 @@from __future__ import annotations+import sysfrom unittest.mock import Mock, callimport pytest@@ -73,6 +74,25 @@ class TestEngineDownloadAsync:[call(original_request), call(redirect_request)])+ @coroutine_test+ async def test_download_async_many_redirects(self, engine):+ """A long chain of requests being replaced by new ones is handled+ iteratively, without hitting the recursion limit."""+ count = sys.getrecursionlimit() * 2+ requests = [Request(f"http://example.com/{i}") for i in range(count)]+ final_response = Response("http://example.com/final", body=b"done")+ engine.downloader.fetch.side_effect = [+ *(defer.succeed(request) for request in requests[1:]),+ defer.succeed(final_response),+ ]+ engine.spider = Mock()+ engine._slot.add_request = Mock()+ engine._slot.remove_request = Mock()++ result = await self._download(engine, requests[0])+ assert result == final_response+ assert engine.downloader.fetch.call_count == count+@coroutine_testasync def test_download_async_no_spider(self, engine):"""Test async download attempt when no spider is available."""diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex a0ae3635b..4f6fa21a0 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -2,6 +2,7 @@ import base64import dataclassesimport loggingimport random+import reimport timefrom abc import ABC, abstractmethodfrom datetime import datetime@@ -19,18 +20,20 @@ import attrimport pytestfrom itemadapter import ItemAdapterfrom twisted.internet.defer import Deferred+from twisted.python.failure import Failure-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.pipelines.files import (+ FileException,FilesPipeline,FSFilesStore,FTPFilesStore,GCSFilesStore,S3FilesStore,)-from scrapy.pipelines.media import MediaPipeline+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFilteredfrom scrapy.settings import Settingsfrom scrapy.utils.asyncio import call_laterfrom scrapy.utils.defer import maybe_deferred_to_future@@ -290,6 +293,71 @@ class TestFilesPipeline:request = Request("http://example.com")assert file_path(request, item=item) == "full/path-to-store-file"+ def test_media_failed_filtered_request(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """A filtered media request (IgnoreRequest) is reported as a+ _MediaRequestFiltered exception and logged at the DEBUG level, instead+ of as a download error with a traceback."""+ request = Request("http://example.com/file.pdf")+ reason = "Filtered offsite request to 'example.com'"+ failure = Failure(IgnoreRequest(reason))++ with (+ caplog.at_level(logging.DEBUG),+ pytest.raises(_MediaRequestFiltered, match=re.escape(reason)),+ ):+ self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)++ assert len(caplog.records) == 1+ record = caplog.records[0]+ assert record.levelname == "DEBUG"+ assert record.exc_info is None+ assert reason in record.getMessage()++ def test_media_failed_download_error(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """A genuine download error is reported as a FileException and logged as+ a warning."""+ request = Request("http://example.com/file.pdf")+ failure = Failure(Exception("boom"))++ with caplog.at_level(logging.WARNING), pytest.raises(FileException):+ self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)++ assert len(caplog.records) == 1+ assert caplog.records[0].levelname == "WARNING"++ @coroutine_test+ async def test_process_item_filtered_request(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """A filtered (e.g. offsite) media request is processed as a failed+ result without being logged as an error with a traceback."""+ item_url = "http://example.com/file.pdf"+ item = _create_item_with_files(item_url)+ request = Request(+ item_url,+ meta={+ "response": IgnoreRequest("Filtered offsite request to 'example.com'")+ },+ )+ with (+ caplog.at_level(logging.DEBUG),+ mock.patch.object(+ FilesPipeline, "get_media_requests", return_value=[request]+ ),+ ):+ result = await self.pipeline.process_item(item)++ assert result["files"] == []+ assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records)+ assert any(+ "Filtered offsite request to 'example.com'" in r.getMessage()+ for r in caplog.records+ )+@pytest.mark.parametrize("bad_type",[diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex ee7a576db..be471e5fe 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,5 +1,6 @@from __future__ import annotations+import loggingfrom unittest.mock import MagicMockimport pytest@@ -10,7 +11,7 @@ from scrapy import signalsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.pipelines.files import FileException-from scrapy.pipelines.media import MediaPipeline+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFilteredfrom scrapy.utils.defer import _defer_sleep_asyncfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_all@@ -152,6 +153,21 @@ class TestBaseMediaPipeline:assert new_item is itemassert len(log.records) == 0+ def test_item_completed_filtered_request_not_logged(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Filtered media requests (e.g. offsite ones) are not logged as errors+ by item_completed(), as they are not download errors."""+ item = {"name": "name"}+ fail = Failure(_MediaRequestFiltered("Filtered offsite request"))+ results = [(True, 1), (False, fail)]++ with caplog.at_level(logging.DEBUG):+ new_item = self.pipe.item_completed(results, item, self.info)++ assert new_item is item+ assert len(caplog.records) == 0+@coroutine_testasync def test_default_process_item(self):item = {"name": "name"}