Scrapy, a fast high-level web crawling & scraping framework for Python.
Infer the FEEDS format from the feed URI's file extension when not set (#8031)
Adrianf44abb91f6a0
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex f18d2743a..7d8a02f8e 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -500,7 +500,9 @@ as a fallback value if that key is not provided for a specific feed definition:- ``format``: the :ref:`serialization format <topics-feed-format>`.- This setting is mandatory, there is no fallback value.+ If not set, it is inferred from the file extension of the feed URI, e.g.+ ``json`` for a URI ending in :file:`.json`. It is mandatory if it cannot+ be inferred this way.- ``batch_item_count``: falls back to:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 874023e25..320106705 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -506,7 +506,7 @@ class FeedExporter:uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)feed_options = {"format": self.settings["FEED_FORMAT"]}self.feeds[uri] = feed_complete_default_values_from_settings(- feed_options, self.settings+ feed_options, self.settings, uri)self.filters[uri] = self._load_filter(feed_options)# End: Backward compatibility for FEED_URI and FEED_FORMAT settings@@ -520,7 +520,7 @@ class FeedExporter:else str(settings_uri))self.feeds[uri] = feed_complete_default_values_from_settings(- feed_options, self.settings+ feed_options, self.settings, uri)self.filters[uri] = self._load_filter(feed_options)@@ -712,7 +712,13 @@ class FeedExporter:def _exporter_supported(self, format_: str) -> bool:if format_ in self.exporters:return True- logger.error("Unknown feed format: %(format)s", {"format": format_})+ if format_:+ logger.error("Unknown feed format: %(format)s", {"format": format_})+ else:+ logger.error(+ "Feed format not set and it could not be inferred from the "+ "feed URI; set it explicitly with the 'format' key"+ )return Falsedef _settings_are_valid(self) -> bool:diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 3b23d2d6c..2b407c8a7 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -127,9 +127,11 @@ def get_sources(use_closest: bool = True) -> list[str]:def feed_complete_default_values_from_settings(- feed: dict[str, Any], settings: BaseSettings+ feed: dict[str, Any], settings: BaseSettings, uri: str | None = None) -> dict[str, Any]:out = feed.copy()+ if uri is not None:+ out.setdefault("format", Path(uri).suffix.removeprefix("."))out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 67de762fc..d75e703f2 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1347,6 +1347,27 @@ class TestFeedExportInit:with pytest.raises(NotConfigured):build_from_crawler(FeedExporter, crawler)+ def test_format_inferred_from_uri(self):+ settings: dict[str, Any] = {+ "FEEDS": {+ "output.json": {},+ },+ }+ crawler = get_crawler(settings_dict=settings)+ exporter = build_from_crawler(FeedExporter, crawler)+ assert exporter.feeds["output.json"]["format"] == "json"++ def test_format_missing_and_not_inferable(self, caplog: pytest.LogCaptureFixture):+ settings: dict[str, Any] = {+ "FEEDS": {+ "stdout:": {},+ },+ }+ crawler = get_crawler(settings_dict=settings)+ with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):+ build_from_crawler(FeedExporter, crawler)+ assert "Feed format not set" in caplog.text+def test_absolute_pathlib_as_uri(self):with tempfile.NamedTemporaryFile(suffix="json") as tmp:settings = {diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 89f5cd141..6c2cda95b 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -176,3 +176,24 @@ class TestFeedExportConfig:"batch_item_count": 2,"item_export_kwargs": {},}++ def test_feed_complete_default_values_from_settings_format_from_uri(self):+ feed: dict[str, Any] = {}+ new_feed = feed_complete_default_values_from_settings(+ feed, Settings(), "output.json"+ )+ assert new_feed["format"] == "json"++ def test_feed_complete_default_values_from_settings_format_kept(self):+ feed = {"format": "csv"}+ new_feed = feed_complete_default_values_from_settings(+ feed, Settings(), "output.json"+ )+ assert new_feed["format"] == "csv"++ def test_feed_complete_default_values_from_settings_format_not_inferable(self):+ feed: dict[str, Any] = {}+ new_feed = feed_complete_default_values_from_settings(+ feed, Settings(), "stdout:"+ )+ assert new_feed["format"] == ""