Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Skip only the misconfigured feed on a bad URI placeholder (#8205)

Adriancd4f21366083
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 127becab7..f1d0d980a 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -552,10 +552,20 @@ class FeedExporter:
def open_spider(self, spider: Spider) -> None:
for uri, feed_options in self.feeds.items():
uri_params = self._get_uri_params(spider, feed_options["uri_params"])
+ try:
+ resolved_uri = apply_uri_params(uri, uri_params)
+ except KeyError as exc:
+ logger.error(
+ f"Feed {uri!r} could not be opened: it contains a "
+ f"placeholder for {exc}, which is not a spider "
+ f"attribute and was not provided by the uri_params "
+ f"function."
+ )
+ continue
self.slots.append(
self._start_new_batch(
batch_id=1,
- uri=apply_uri_params(uri, uri_params),
+ uri=resolved_uri,
feed_options=feed_options,
spider=spider,
uri_template=uri,
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d75e703f2..08ba230f8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1299,6 +1299,28 @@ class TestFeedExporterSignals:
assert self.feed_exporter_closed_received
+class TestFeedExporterOpenSpider:
+ @coroutine_test
+ async def test_bad_uri_placeholder_skips_only_that_feed(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ with tempfile.NamedTemporaryFile(suffix="json") as tmp:
+ settings = {
+ "FEEDS": {
+ printf_escape(path_to_url(tmp.name)): {"format": "json"},
+ "file:///nonexistent/%(undefined_attr)s.json": {"format": "json"},
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ feed_exporter = build_from_crawler(FeedExporter, crawler)
+ spider = scrapy.Spider.from_crawler(crawler, "default")
+ with caplog.at_level(logging.ERROR):
+ feed_exporter.open_spider(spider)
+ assert len(feed_exporter.slots) == 1
+ assert "undefined_attr" in caplog.text
+ await feed_exporter.close_spider(spider)
+
+
class TestItemFilter:
def test_no_feed_options(self):
item_filter = ItemFilter(None)
diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py
index ce97286d7..16d115a60 100644
--- a/tests/test_feedexport_uri_params.py
+++ b/tests/test_feedexport_uri_params.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import logging
import warnings
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
@@ -72,7 +73,7 @@ class TestURIParams(ABC):
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
- def test_empty_dict(self):
+ def test_empty_dict(self, caplog: pytest.LogCaptureFixture) -> None:
def uri_params(params, spider):
return {}
@@ -85,9 +86,12 @@ class TestURIParams(ABC):
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
- with pytest.raises(KeyError):
+ with caplog.at_level(logging.ERROR):
feed_exporter.open_spider(spider)
+ assert feed_exporter.slots == []
+ assert "'name'" in caplog.text
+
def test_params_as_is(self):
def uri_params(params, spider):
return params