Scrapy, a fast high-level web crawling & scraping framework for Python.
Skip only the misconfigured feed on a bad URI placeholder (#8205)
Adriancd4f21366083
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 127becab7..f1d0d980a 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -552,10 +552,20 @@ class FeedExporter:def open_spider(self, spider: Spider) -> None:for uri, feed_options in self.feeds.items():uri_params = self._get_uri_params(spider, feed_options["uri_params"])+ try:+ resolved_uri = apply_uri_params(uri, uri_params)+ except KeyError as exc:+ logger.error(+ f"Feed {uri!r} could not be opened: it contains a "+ f"placeholder for {exc}, which is not a spider "+ f"attribute and was not provided by the uri_params "+ f"function."+ )+ continueself.slots.append(self._start_new_batch(batch_id=1,- uri=apply_uri_params(uri, uri_params),+ uri=resolved_uri,feed_options=feed_options,spider=spider,uri_template=uri,diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex d75e703f2..08ba230f8 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1299,6 +1299,28 @@ class TestFeedExporterSignals:assert self.feed_exporter_closed_received+class TestFeedExporterOpenSpider:+ @coroutine_test+ async def test_bad_uri_placeholder_skips_only_that_feed(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ with tempfile.NamedTemporaryFile(suffix="json") as tmp:+ settings = {+ "FEEDS": {+ printf_escape(path_to_url(tmp.name)): {"format": "json"},+ "file:///nonexistent/%(undefined_attr)s.json": {"format": "json"},+ },+ }+ crawler = get_crawler(settings_dict=settings)+ feed_exporter = build_from_crawler(FeedExporter, crawler)+ spider = scrapy.Spider.from_crawler(crawler, "default")+ with caplog.at_level(logging.ERROR):+ feed_exporter.open_spider(spider)+ assert len(feed_exporter.slots) == 1+ assert "undefined_attr" in caplog.text+ await feed_exporter.close_spider(spider)++class TestItemFilter:def test_no_feed_options(self):item_filter = ItemFilter(None)diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.pyindex ce97286d7..16d115a60 100644--- a/tests/test_feedexport_uri_params.py+++ b/tests/test_feedexport_uri_params.py@@ -1,5 +1,6 @@from __future__ import annotations+import loggingimport warningsfrom abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, Any@@ -72,7 +73,7 @@ class TestURIParams(ABC):assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"- def test_empty_dict(self):+ def test_empty_dict(self, caplog: pytest.LogCaptureFixture) -> None:def uri_params(params, spider):return {}@@ -85,9 +86,12 @@ class TestURIParams(ABC):with warnings.catch_warnings():warnings.simplefilter("error", ScrapyDeprecationWarning)- with pytest.raises(KeyError):+ with caplog.at_level(logging.ERROR):feed_exporter.open_spider(spider)+ assert feed_exporter.slots == []+ assert "'name'" in caplog.text+def test_params_as_is(self):def uri_params(params, spider):return params