Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn when feeds are configured but FeedExporter is disabled (#7902)
Adrian60b1badacd19
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex d9c919db9..bb602784b 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -6,6 +6,7 @@ from __future__ import annotationsimport argparseimport builtins+import loggingimport osimport warningsfrom abc import ABC, abstractmethod@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom twisted.python import failure+from scrapy import signalsfrom scrapy.exceptions import ScrapyDeprecationWarning, UsageError+from scrapy.extensions.feedexport import FeedExporterfrom scrapy.utils.conf import arglist_to_dict, feed_process_params_from_clifrom scrapy.utils.deprecate import method_is_overriddenfrom scrapy.utils.python import global_object_name@@ -22,10 +25,14 @@ from scrapy.utils.python import global_object_nameif TYPE_CHECKING:from collections.abc import Iterable+ from scrapy import Spiderfrom scrapy.crawler import Crawler, CrawlerProcessBasefrom scrapy.settings import Settings+logger = logging.getLogger(__name__)++class ScrapyCommand(ABC):requires_project: bool = Falserequires_crawler_process: bool = True@@ -206,6 +213,24 @@ class BaseRunSpiderCommand(ScrapyCommand):)self.settings.set("FEEDS", feeds, priority="cmdline")+ def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:+ assert self.crawler_process is not None+ crawler = self.crawler_process.create_crawler(spidercls)+ crawler.signals.connect(+ self._warn_if_feeds_unused, signal=signals.engine_started+ )+ return crawler++ def _warn_if_feeds_unused(self, sender: Crawler, **kwargs: Any) -> None:+ if (+ sender.settings.getdict("FEEDS")+ and sender.get_extension(FeedExporter) is None+ ):+ logger.warning(+ "The FeedExporter extension is not enabled, so no item will be "+ "exported to the configured feeds (FEEDS, -o, -O)."+ )+class ScrapyHelpFormatter(argparse.HelpFormatter):"""diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.pyindex 4e086e057..7cc761142 100644--- a/scrapy/commands/crawl.py+++ b/scrapy/commands/crawl.py@@ -28,7 +28,7 @@ class Command(BaseRunSpiderCommand):spname = args[0]assert self.crawler_process- self.crawler_process.crawl(spname, **opts.spargs)+ self.crawler_process.crawl(self._create_crawler(spname), **opts.spargs)self.crawler_process.start()if self.crawler_process.bootstrap_failed:self.exitcode = 1diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 5ef83ab6f..d50075fb5 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -266,8 +266,8 @@ class Command(BaseRunSpiderCommand):def start_parsing(self, url: str, opts: argparse.Namespace) -> None:assert self.crawler_processassert self.spidercls- self.crawler_process.crawl(self.spidercls, **opts.spargs)- self.pcrawler = next(iter(self.crawler_process.crawlers))+ self.pcrawler = self._create_crawler(self.spidercls)+ self.crawler_process.crawl(self.pcrawler, **opts.spargs)self.crawler_process.start()if self.crawler_process.bootstrap_failed:self.exitcode = 1diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 9cdb393ab..09d268f1e 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -59,7 +59,7 @@ class Command(BaseRunSpiderCommand):spidercls = spclasses.pop()assert self.crawler_process- self.crawler_process.crawl(spidercls, **opts.spargs)+ self.crawler_process.crawl(self._create_crawler(spidercls), **opts.spargs)self.crawler_process.start()if self.crawler_process.bootstrap_failed:diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 73a85dd5d..fa1763d21 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -314,6 +314,55 @@ class MySpider(scrapy.Spider):log = self.get_log(tmp_path, spider_code, args=args)assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log+ def test_output_feed_exporter_disabled(self, tmp_path: Path) -> None:+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ custom_settings = {+ "EXTENSIONS": {"scrapy.extensions.feedexport.FeedExporter": None},+ }++ start_urls = ["data:,"]++ def parse(self, response):+ yield {"hello": "world"}+"""+ args = ["-o", "example.json"]+ log = self.get_log(tmp_path, spider_code, args=args)+ assert "FeedExporter extension is not enabled" in log+ assert not (tmp_path / "example.json").exists()++ def test_output_feed_exporter_subclass(self, tmp_path: Path) -> None:+ spider_code = """+import scrapy+from scrapy.extensions.feedexport import FeedExporter++class MyFeedExporter(FeedExporter):+ pass++class MySpider(scrapy.Spider):+ name = 'myspider'++ custom_settings = {+ "EXTENSIONS": {+ "scrapy.extensions.feedexport.FeedExporter": None,+ MyFeedExporter: 0,+ },+ }++ start_urls = ["data:,"]++ def parse(self, response):+ yield {"hello": "world"}+"""+ args = ["-o", "example.json"]+ log = self.get_log(tmp_path, spider_code, args=args)+ assert "FeedExporter extension is not enabled" not in log+ assert (tmp_path / "example.json").exists()+@pytest.mark.parametrize("arg", ["output.json:json", "output.json"])def test_absolute_path(self, tmp_path: Path, arg: str) -> None:spider_code = """