Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Infer the FEEDS format from the feed URI's file extension when not set (#8031)

Adrianf44abb91f6a0
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index f18d2743a..7d8a02f8e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -500,7 +500,9 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``format``: the :ref:`serialization format <topics-feed-format>`.
- This setting is mandatory, there is no fallback value.
+ If not set, it is inferred from the file extension of the feed URI, e.g.
+ ``json`` for a URI ending in :file:`.json`. It is mandatory if it cannot
+ be inferred this way.
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 874023e25..320106705 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -506,7 +506,7 @@ class FeedExporter:
uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
feed_options = {"format": self.settings["FEED_FORMAT"]}
self.feeds[uri] = feed_complete_default_values_from_settings(
- feed_options, self.settings
+ feed_options, self.settings, uri
)
self.filters[uri] = self._load_filter(feed_options)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
@@ -520,7 +520,7 @@ class FeedExporter:
else str(settings_uri)
)
self.feeds[uri] = feed_complete_default_values_from_settings(
- feed_options, self.settings
+ feed_options, self.settings, uri
)
self.filters[uri] = self._load_filter(feed_options)
@@ -712,7 +712,13 @@ class FeedExporter:
def _exporter_supported(self, format_: str) -> bool:
if format_ in self.exporters:
return True
- logger.error("Unknown feed format: %(format)s", {"format": format_})
+ if format_:
+ logger.error("Unknown feed format: %(format)s", {"format": format_})
+ else:
+ logger.error(
+ "Feed format not set and it could not be inferred from the "
+ "feed URI; set it explicitly with the 'format' key"
+ )
return False
def _settings_are_valid(self) -> bool:
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 3b23d2d6c..2b407c8a7 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -127,9 +127,11 @@ def get_sources(use_closest: bool = True) -> list[str]:
def feed_complete_default_values_from_settings(
- feed: dict[str, Any], settings: BaseSettings
+ feed: dict[str, Any], settings: BaseSettings, uri: str | None = None
) -> dict[str, Any]:
out = feed.copy()
+ if uri is not None:
+ out.setdefault("format", Path(uri).suffix.removeprefix("."))
out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 67de762fc..d75e703f2 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1347,6 +1347,27 @@ class TestFeedExportInit:
with pytest.raises(NotConfigured):
build_from_crawler(FeedExporter, crawler)
+ def test_format_inferred_from_uri(self):
+ settings: dict[str, Any] = {
+ "FEEDS": {
+ "output.json": {},
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ exporter = build_from_crawler(FeedExporter, crawler)
+ assert exporter.feeds["output.json"]["format"] == "json"
+
+ def test_format_missing_and_not_inferable(self, caplog: pytest.LogCaptureFixture):
+ settings: dict[str, Any] = {
+ "FEEDS": {
+ "stdout:": {},
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):
+ build_from_crawler(FeedExporter, crawler)
+ assert "Feed format not set" in caplog.text
+
def test_absolute_pathlib_as_uri(self):
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
settings = {
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 89f5cd141..6c2cda95b 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -176,3 +176,24 @@ class TestFeedExportConfig:
"batch_item_count": 2,
"item_export_kwargs": {},
}
+
+ def test_feed_complete_default_values_from_settings_format_from_uri(self):
+ feed: dict[str, Any] = {}
+ new_feed = feed_complete_default_values_from_settings(
+ feed, Settings(), "output.json"
+ )
+ assert new_feed["format"] == "json"
+
+ def test_feed_complete_default_values_from_settings_format_kept(self):
+ feed = {"format": "csv"}
+ new_feed = feed_complete_default_values_from_settings(
+ feed, Settings(), "output.json"
+ )
+ assert new_feed["format"] == "csv"
+
+ def test_feed_complete_default_values_from_settings_format_not_inferable(self):
+ feed: dict[str, Any] = {}
+ new_feed = feed_complete_default_values_from_settings(
+ feed, Settings(), "stdout:"
+ )
+ assert new_feed["format"] == ""