Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Warn when feeds are configured but FeedExporter is disabled (#7902)

Adrian60b1badacd19
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index d9c919db9..bb602784b 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -6,6 +6,7 @@ from __future__ import annotations
import argparse
import builtins
+import logging
import os
import warnings
from abc import ABC, abstractmethod
@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, ClassVar
from twisted.python import failure
+from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
+from scrapy.extensions.feedexport import FeedExporter
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.python import global_object_name
@@ -22,10 +25,14 @@ from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Iterable
+ from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerProcessBase
from scrapy.settings import Settings
+logger = logging.getLogger(__name__)
+
+
class ScrapyCommand(ABC):
requires_project: bool = False
requires_crawler_process: bool = True
@@ -206,6 +213,24 @@ class BaseRunSpiderCommand(ScrapyCommand):
)
self.settings.set("FEEDS", feeds, priority="cmdline")
+ def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:
+ assert self.crawler_process is not None
+ crawler = self.crawler_process.create_crawler(spidercls)
+ crawler.signals.connect(
+ self._warn_if_feeds_unused, signal=signals.engine_started
+ )
+ return crawler
+
+ def _warn_if_feeds_unused(self, sender: Crawler, **kwargs: Any) -> None:
+ if (
+ sender.settings.getdict("FEEDS")
+ and sender.get_extension(FeedExporter) is None
+ ):
+ logger.warning(
+ "The FeedExporter extension is not enabled, so no item will be "
+ "exported to the configured feeds (FEEDS, -o, -O)."
+ )
+
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 4e086e057..7cc761142 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -28,7 +28,7 @@ class Command(BaseRunSpiderCommand):
spname = args[0]
assert self.crawler_process
- self.crawler_process.crawl(spname, **opts.spargs)
+ self.crawler_process.crawl(self._create_crawler(spname), **opts.spargs)
self.crawler_process.start()
if self.crawler_process.bootstrap_failed:
self.exitcode = 1
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 5ef83ab6f..d50075fb5 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -266,8 +266,8 @@ class Command(BaseRunSpiderCommand):
def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
assert self.crawler_process
assert self.spidercls
- self.crawler_process.crawl(self.spidercls, **opts.spargs)
- self.pcrawler = next(iter(self.crawler_process.crawlers))
+ self.pcrawler = self._create_crawler(self.spidercls)
+ self.crawler_process.crawl(self.pcrawler, **opts.spargs)
self.crawler_process.start()
if self.crawler_process.bootstrap_failed:
self.exitcode = 1
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 9cdb393ab..09d268f1e 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -59,7 +59,7 @@ class Command(BaseRunSpiderCommand):
spidercls = spclasses.pop()
assert self.crawler_process
- self.crawler_process.crawl(spidercls, **opts.spargs)
+ self.crawler_process.crawl(self._create_crawler(spidercls), **opts.spargs)
self.crawler_process.start()
if self.crawler_process.bootstrap_failed:
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 73a85dd5d..fa1763d21 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -314,6 +314,55 @@ class MySpider(scrapy.Spider):
log = self.get_log(tmp_path, spider_code, args=args)
assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log
+ def test_output_feed_exporter_disabled(self, tmp_path: Path) -> None:
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ custom_settings = {
+ "EXTENSIONS": {"scrapy.extensions.feedexport.FeedExporter": None},
+ }
+
+ start_urls = ["data:,"]
+
+ def parse(self, response):
+ yield {"hello": "world"}
+"""
+ args = ["-o", "example.json"]
+ log = self.get_log(tmp_path, spider_code, args=args)
+ assert "FeedExporter extension is not enabled" in log
+ assert not (tmp_path / "example.json").exists()
+
+ def test_output_feed_exporter_subclass(self, tmp_path: Path) -> None:
+ spider_code = """
+import scrapy
+from scrapy.extensions.feedexport import FeedExporter
+
+class MyFeedExporter(FeedExporter):
+ pass
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ custom_settings = {
+ "EXTENSIONS": {
+ "scrapy.extensions.feedexport.FeedExporter": None,
+ MyFeedExporter: 0,
+ },
+ }
+
+ start_urls = ["data:,"]
+
+ def parse(self, response):
+ yield {"hello": "world"}
+"""
+ args = ["-o", "example.json"]
+ log = self.get_log(tmp_path, spider_code, args=args)
+ assert "FeedExporter extension is not enabled" not in log
+ assert (tmp_path / "example.json").exists()
+
@pytest.mark.parametrize("arg", ["output.json:json", "output.json"])
def test_absolute_path(self, tmp_path: Path, arg: str) -> None:
spider_code = """