Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move from optparse to argparse (#5374)

Alexe2e2ffd0d162
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 91482ce01..491c4beab 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,13 +1,13 @@
import sys
import os
-import optparse
+import argparse
import cProfile
import inspect
import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
@@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
- parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
- conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
- parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
- parser.description = cmd.long_desc()
+ parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
+ usage=f"scrapy {cmdname} {cmd.syntax()}",
+ conflict_handler='resolve',
+ description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
cmd.add_options(parser)
- opts, args = parser.parse_args(args=argv[1:])
+ opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings)
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 5f1dabd33..fb304b8c0 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -2,7 +2,7 @@
Base class for Scrapy commands
"""
import os
-from optparse import OptionGroup
+import argparse
from typing import Any, Dict
from twisted.python import failure
@@ -59,22 +59,20 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
- group = OptionGroup(parser, "Global Options")
- group.add_option("--logfile", metavar="FILE",
- help="log file. if omitted stderr will be used")
- group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
- help=f"log level (default: {self.settings['LOG_LEVEL']})")
- group.add_option("--nolog", action="store_true",
- help="disable logging completely")
- group.add_option("--profile", metavar="FILE", default=None,
- help="write python cProfile stats to FILE")
- group.add_option("--pidfile", metavar="FILE",
- help="write process ID to FILE")
- group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
- help="set/override setting (may be repeated)")
- group.add_option("--pdb", action="store_true", help="enable pdb on failure")
-
- parser.add_option_group(group)
+ group = parser.add_argument_group(title='Global Options')
+ group.add_argument("--logfile", metavar="FILE",
+ help="log file. if omitted stderr will be used")
+ group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
+ help=f"log level (default: {self.settings['LOG_LEVEL']})")
+ group.add_argument("--nolog", action="store_true",
+ help="disable logging completely")
+ group.add_argument("--profile", metavar="FILE", default=None,
+ help="write python cProfile stats to FILE")
+ group.add_argument("--pidfile", metavar="FILE",
+ help="write process ID to FILE")
+ group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
+ help="set/override setting (may be repeated)")
+ group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
@@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
- help="set spider argument (may be repeated)")
- parser.add_option("-o", "--output", metavar="FILE", action="append",
- help="append scraped items to the end of FILE (use - for stdout)")
- parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
- help="dump scraped items into FILE, overwriting any existing file")
- parser.add_option("-t", "--output-format", metavar="FORMAT",
- help="format to use for dumping items")
+ parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
+ help="set spider argument (may be repeated)")
+ parser.add_argument("-o", "--output", metavar="FILE", action="append",
+ help="append scraped items to the end of FILE (use - for stdout)")
+ parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
+ help="dump scraped items into FILE, overwriting any existing file")
+ parser.add_argument("-t", "--output-format", metavar="FORMAT",
+ help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
+
+
+class ScrapyHelpFormatter(argparse.HelpFormatter):
+ """
+ Help Formatter for scrapy command line help messages.
+ """
+ def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
+ super().__init__(prog, indent_increment=indent_increment,
+ max_help_position=max_help_position, width=width)
+
+ def _join_parts(self, part_strings):
+ parts = self.format_part_strings(part_strings)
+ return super()._join_parts(parts)
+
+ def format_part_strings(self, part_strings):
+ """
+ Underline and title case command line help message headers.
+ """
+ if part_strings and part_strings[0].startswith("usage: "):
+ part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
+ headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
+ for index in headings[::-1]:
+ char = '-' if "Global Options" in part_strings[index] else '='
+ part_strings[index] = part_strings[index][:-2].title()
+ underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
+ part_strings.insert(index + 1, underline)
+ return part_strings
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index ae21d86e6..a16f4beb7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -49,10 +49,10 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("-l", "--list", dest="list", action="store_true",
- help="only list contracts, without checking them")
- parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
- help="print contract tests for all spiders")
+ parser.add_argument("-l", "--list", dest="list", action="store_true",
+ help="only list contracts, without checking them")
+ parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
+ help="print contract tests for all spiders")
def run(self, args, opts):
# load contracts
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 95f87e8c3..9b2ebb37f 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -26,11 +26,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--spider", dest="spider", help="use this spider")
- parser.add_option("--headers", dest="headers", action="store_true",
- help="print response HTTP headers instead of body")
- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
- help="do not handle HTTP 3xx status codes and print response as-is")
+ parser.add_argument("--spider", dest="spider", help="use this spider")
+ parser.add_argument("--headers", dest="headers", action="store_true",
+ help="print response HTTP headers instead of body")
+ parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
+ help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2082a4974..ed5f588e9 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -44,16 +44,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("-l", "--list", dest="list", action="store_true",
- help="List available templates")
- parser.add_option("-e", "--edit", dest="edit", action="store_true",
- help="Edit spider after creating it")
- parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
- help="Dump template to standard output")
- parser.add_option("-t", "--template", dest="template", default="basic",
- help="Uses a custom template.")
- parser.add_option("--force", dest="force", action="store_true",
- help="If the spider already exists, overwrite it with the template")
+ parser.add_argument("-l", "--list", dest="list", action="store_true",
+ help="List available templates")
+ parser.add_argument("-e", "--edit", dest="edit", action="store_true",
+ help="Edit spider after creating it")
+ parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
+ help="Dump template to standard output")
+ parser.add_argument("-t", "--template", dest="template", default="basic",
+ help="Uses a custom template.")
+ parser.add_argument("--force", dest="force", action="store_true",
+ help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 52118db1b..a3f6b96f4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
- parser.add_option("--spider", dest="spider", default=None,
- help="use this spider without looking for one")
- parser.add_option("--pipelines", action="store_true",
- help="process items through pipelines")
- parser.add_option("--nolinks", dest="nolinks", action="store_true",
- help="don't show links to follow (extracted requests)")
- parser.add_option("--noitems", dest="noitems", action="store_true",
- help="don't show scraped items")
- parser.add_option("--nocolour", dest="nocolour", action="store_true",
- help="avoid using pygments to colorize the output")
- parser.add_option("-r", "--rules", dest="rules", action="store_true",
- help="use CrawlSpider rules to discover the callback")
- parser.add_option("-c", "--callback", dest="callback",
- help="use this callback for parsing, instead looking for a callback")
- parser.add_option("-m", "--meta", dest="meta",
- help="inject extra meta into the Request, it must be a valid raw json string")
- parser.add_option("--cbkwargs", dest="cbkwargs",
- help="inject extra callback kwargs into the Request, it must be a valid raw json string")
- parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
- help="maximum depth for parsing requests [default: %default]")
- parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
- help="print each depth level one by one")
+ parser.add_argument("--spider", dest="spider", default=None,
+ help="use this spider without looking for one")
+ parser.add_argument("--pipelines", action="store_true",
+ help="process items through pipelines")
+ parser.add_argument("--nolinks", dest="nolinks", action="store_true",
+ help="don't show links to follow (extracted requests)")
+ parser.add_argument("--noitems", dest="noitems", action="store_true",
+ help="don't show scraped items")
+ parser.add_argument("--nocolour", dest="nocolour", action="store_true",
+ help="avoid using pygments to colorize the output")
+ parser.add_argument("-r", "--rules", dest="rules", action="store_true",
+ help="use CrawlSpider rules to discover the callback")
+ parser.add_argument("-c", "--callback", dest="callback",
+ help="use this callback for parsing, instead looking for a callback")
+ parser.add_argument("-m", "--meta", dest="meta",
+ help="inject extra meta into the Request, it must be a valid raw json string")
+ parser.add_argument("--cbkwargs", dest="cbkwargs",
+ help="inject extra callback kwargs into the Request, it must be a valid raw json string")
+ parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
+ help="maximum depth for parsing requests [default: %default]")
+ parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+ help="print each depth level one by one")
@property
def max_level(self):
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 8d49e440f..1b2e2601e 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -18,16 +18,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--get", dest="get", metavar="SETTING",
- help="print raw setting value")
- parser.add_option("--getbool", dest="getbool", metavar="SETTING",
- help="print setting value, interpreted as a boolean")
- parser.add_option("--getint", dest="getint", metavar="SETTING",
- help="print setting value, interpreted as an integer")
- parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
- help="print setting value, interpreted as a float")
- parser.add_option("--getlist", dest="getlist", metavar="SETTING",
- help="print setting value, interpreted as a list")
+ parser.add_argument("--get", dest="get", metavar="SETTING",
+ help="print raw setting value")
+ parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
+ help="print setting value, interpreted as a boolean")
+ parser.add_argument("--getint", dest="getint", metavar="SETTING",
+ help="print setting value, interpreted as an integer")
+ parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
+ help="print setting value, interpreted as a float")
+ parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
+ help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index de81986d8..f67a5886a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -33,12 +33,12 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("-c", dest="code",
- help="evaluate the code in the shell, print the result and exit")
- parser.add_option("--spider", dest="spider",
- help="use this spider")
- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
- help="do not handle HTTP 3xx status codes and print response as-is")
+ parser.add_argument("-c", dest="code",
+ help="evaluate the code in the shell, print the result and exit")
+ parser.add_argument("--spider", dest="spider",
+ help="use this spider")
+ parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
+ help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 1237610cb..c6a3c273a 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -16,8 +16,8 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
- help="also display twisted/python/platform info (useful for bug reports)")
+ parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
+ help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index c8f873334..b1f52abe2 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,3 +1,4 @@
+import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
@@ -12,7 +13,7 @@ class Command(fetch.Command):
def add_options(self, parser):
super().add_options(parser)
- parser.remove_option("--headers")
+ parser.add_argument('--headers', help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index ed3848d88..f21ee971d 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,9 @@
import os
+import argparse
from os.path import join, abspath, isfile, exists
from twisted.internet import defer
+from scrapy.commands import parse
+from scrapy.settings import Settings
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
from scrapy.utils.python import to_unicode
@@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
content = '[\n{},\n{"foo": "bar"}\n]'
with open(file_path, 'r') as f:
self.assertEqual(f.read(), content)
+
+ def test_parse_add_options(self):
+ command = parse.Command()
+ command.settings = Settings()
+ parser = argparse.ArgumentParser(
+ prog='scrapy', formatter_class=argparse.HelpFormatter,
+ conflict_handler='resolve', prefix_chars='-'
+ )
+ command.add_options(parser)
+ namespace = parser.parse_args(
+ ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
+ )
+ self.assertTrue(namespace.nolinks)
+ self.assertEqual(namespace.depth, 2)
+ self.assertEqual(namespace.spider, self.spider_name)
+ self.assertTrue(namespace.verbose)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7473b53df..7cd19b29a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -1,6 +1,6 @@
import inspect
import json
-import optparse
+import argparse
import os
import platform
import re
@@ -23,7 +23,7 @@ from twisted.python.versions import Version
from twisted.trial import unittest
import scrapy
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
from scrapy.commands.startproject import IGNORE
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
@@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
def setUp(self):
self.command = ScrapyCommand()
self.command.settings = Settings()
- self.parser = optparse.OptionParser(
- formatter=optparse.TitledHelpFormatter(),
- conflict_handler='resolve',
- )
+ self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
+ conflict_handler='resolve')
self.command.add_options(self.parser)
def test_settings_json_string(self):
feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
- opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
+ opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
self.command.process_options(args, opts)
self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
+ def test_help_formatter(self):
+ formatter = ScrapyHelpFormatter(prog='scrapy')
+ part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
+ '\n', 'optional arguments:\n', '\n', 'Global Options:\n']
+ self.assertEqual(
+ formatter._join_parts(part_strings),
+ ('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'
+ 'Optional Arguments\n==================\n\n'
+ 'Global Options\n--------------\n')
+ )
+
class ProjectTest(unittest.TestCase):
project_name = 'testproject'
@@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):
self.assertNotIn('Unhandled Error', log)
+class ViewCommandTest(CommandTest):
+
+ def test_methods(self):
+ command = view.Command()
+ command.settings = Settings()
+ parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
+ formatter_class=ScrapyHelpFormatter,
+ conflict_handler='resolve')
+ command.add_options(parser)
+ self.assertEqual(command.short_desc(),
+ "Open URL in browser, as seen by Scrapy")
+ self.assertIn("URL using the Scrapy downloader and show its",
+ command.long_desc())
+
+
class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()):