Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Prevent running the -O and -t command-line options together (#5605)

Magnus Offermannsfd692f309105
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 8c0b8e55f..362190116 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -271,11 +271,31 @@ crawl
Start crawling using a spider.
+Supported options:
+
+* ``-h, --help``: show a help message and exit
+
+* ``-a NAME=VALUE``: set a spider argument (may be repeated)
+
+* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
+
+* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
+
+* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
+
Usage examples::
$ scrapy crawl myspider
[ ... myspider starts crawling ... ]
+ $ scrapy -o myfile:csv myspider
+ [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
+
+ $ scrapy -O myfile:json myspider
+ [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
+
+ $ scrapy -o myfile -t csv myspider
+ [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index fb304b8c0..49c4e2f42 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
- help="append scraped items to the end of FILE (use - for stdout)")
+ help="append scraped items to the end of FILE (use - for stdout),"
+ " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
- help="dump scraped items into FILE, overwriting any existing file")
+ help="dump scraped items into FILE, overwriting any existing file,"
+ " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 00cc53725..6404edda6 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -155,6 +155,15 @@ def feed_process_params_from_cli(settings, output, output_format=None,
raise UsageError(
"Please use only one of -o/--output and -O/--overwrite-output"
)
+ if output_format:
+ raise UsageError(
+ "-t/--output-format is a deprecated command line option"
+ " and does not work in combination with -O/--overwrite-output."
+ " To specify a format please specify it after a colon at the end of the"
+ " output URI (i.e. -O <URI>:<FORMAT>)."
+ " Example working in the tutorial: "
+ "scrapy crawl quotes -O quotes.json:json"
+ )
output = overwrite_output
overwrite = True
@@ -162,9 +171,13 @@ def feed_process_params_from_cli(settings, output, output_format=None,
if len(output) == 1:
check_valid_format(output_format)
message = (
- 'The -t command line option is deprecated in favor of '
- 'specifying the output format within the output URI. See the '
- 'documentation of the -o and -O options for more information.'
+ "The -t/--output-format command line option is deprecated in favor of "
+ "specifying the output format within the output URI using the -o/--output or the"
+ " -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"
+ " of the -o or -O option or the following examples for more information. "
+ "Examples working in the tutorial: "
+ "scrapy crawl quotes -o quotes.csv:csv or "
+ "scrapy crawl quotes -O quotes.json:json"
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {'format': output_format}}