Scrapy, a fast high-level web crawling & scraping framework for Python.
Move from optparse to argparse (#5374)
Alexe2e2ffd0d162
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex 91482ce01..491c4beab 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -1,13 +1,13 @@import sysimport os-import optparse+import argparseimport cProfileimport inspectimport pkg_resourcesimport scrapyfrom scrapy.crawler import CrawlerProcess-from scrapy.commands import ScrapyCommand+from scrapy.commands import ScrapyCommand, ScrapyHelpFormatterfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.misc import walk_modulesfrom scrapy.utils.project import inside_project, get_project_settings@@ -123,8 +123,6 @@ def execute(argv=None, settings=None):inproject = inside_project()cmds = _get_commands_dict(settings, inproject)cmdname = _pop_command_name(argv)- parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),- conflict_handler='resolve')if not cmdname:_print_commands(settings, inproject)sys.exit(0)@@ -133,12 +131,14 @@ def execute(argv=None, settings=None):sys.exit(2)cmd = cmds[cmdname]- parser.usage = f"scrapy {cmdname} {cmd.syntax()}"- parser.description = cmd.long_desc()+ parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,+ usage=f"scrapy {cmdname} {cmd.syntax()}",+ conflict_handler='resolve',+ description=cmd.long_desc())settings.setdict(cmd.default_settings, priority='command')cmd.settings = settingscmd.add_options(parser)- opts, args = parser.parse_args(args=argv[1:])+ opts, args = parser.parse_known_args(args=argv[1:])_run_print_help(parser, cmd.process_options, args, opts)cmd.crawler_process = CrawlerProcess(settings)diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 5f1dabd33..fb304b8c0 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -2,7 +2,7 @@Base class for Scrapy commands"""import os-from optparse import OptionGroup+import argparsefrom typing import Any, Dictfrom twisted.python import failure@@ -59,22 +59,20 @@ class ScrapyCommand:"""Populate option parse with options available for this command"""- group = OptionGroup(parser, "Global Options")- group.add_option("--logfile", metavar="FILE",- help="log file. if omitted stderr will be used")- group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,- help=f"log level (default: {self.settings['LOG_LEVEL']})")- group.add_option("--nolog", action="store_true",- help="disable logging completely")- group.add_option("--profile", metavar="FILE", default=None,- help="write python cProfile stats to FILE")- group.add_option("--pidfile", metavar="FILE",- help="write process ID to FILE")- group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",- help="set/override setting (may be repeated)")- group.add_option("--pdb", action="store_true", help="enable pdb on failure")-- parser.add_option_group(group)+ group = parser.add_argument_group(title='Global Options')+ group.add_argument("--logfile", metavar="FILE",+ help="log file. if omitted stderr will be used")+ group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,+ help=f"log level (default: {self.settings['LOG_LEVEL']})")+ group.add_argument("--nolog", action="store_true",+ help="disable logging completely")+ group.add_argument("--profile", metavar="FILE", default=None,+ help="write python cProfile stats to FILE")+ group.add_argument("--pidfile", metavar="FILE",+ help="write process ID to FILE")+ group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",+ help="set/override setting (may be repeated)")+ group.add_argument("--pdb", action="store_true", help="enable pdb on failure")def process_options(self, args, opts):try:@@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):"""def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",- help="set spider argument (may be repeated)")- parser.add_option("-o", "--output", metavar="FILE", action="append",- help="append scraped items to the end of FILE (use - for stdout)")- parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",- help="dump scraped items into FILE, overwriting any existing file")- parser.add_option("-t", "--output-format", metavar="FORMAT",- help="format to use for dumping items")+ parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",+ help="set spider argument (may be repeated)")+ parser.add_argument("-o", "--output", metavar="FILE", action="append",+ help="append scraped items to the end of FILE (use - for stdout)")+ parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",+ help="dump scraped items into FILE, overwriting any existing file")+ parser.add_argument("-t", "--output-format", metavar="FORMAT",+ help="format to use for dumping items")def process_options(self, args, opts):ScrapyCommand.process_options(self, args, opts)@@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):opts.overwrite_output,)self.settings.set('FEEDS', feeds, priority='cmdline')+++class ScrapyHelpFormatter(argparse.HelpFormatter):+ """+ Help Formatter for scrapy command line help messages.+ """+ def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):+ super().__init__(prog, indent_increment=indent_increment,+ max_help_position=max_help_position, width=width)++ def _join_parts(self, part_strings):+ parts = self.format_part_strings(part_strings)+ return super()._join_parts(parts)++ def format_part_strings(self, part_strings):+ """+ Underline and title case command line help message headers.+ """+ if part_strings and part_strings[0].startswith("usage: "):+ part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]+ headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]+ for index in headings[::-1]:+ char = '-' if "Global Options" in part_strings[index] else '='+ part_strings[index] = part_strings[index][:-2].title()+ underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])+ part_strings.insert(index + 1, underline)+ return part_stringsdiff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex ae21d86e6..a16f4beb7 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -49,10 +49,10 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("-l", "--list", dest="list", action="store_true",- help="only list contracts, without checking them")- parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',- help="print contract tests for all spiders")+ parser.add_argument("-l", "--list", dest="list", action="store_true",+ help="only list contracts, without checking them")+ parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',+ help="print contract tests for all spiders")def run(self, args, opts):# load contractsdiff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex 95f87e8c3..9b2ebb37f 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -26,11 +26,11 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("--spider", dest="spider", help="use this spider")- parser.add_option("--headers", dest="headers", action="store_true",- help="print response HTTP headers instead of body")- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,- help="do not handle HTTP 3xx status codes and print response as-is")+ parser.add_argument("--spider", dest="spider", help="use this spider")+ parser.add_argument("--headers", dest="headers", action="store_true",+ help="print response HTTP headers instead of body")+ parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,+ help="do not handle HTTP 3xx status codes and print response as-is")def _print_headers(self, headers, prefix):for key, values in headers.items():diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 2082a4974..ed5f588e9 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -44,16 +44,16 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("-l", "--list", dest="list", action="store_true",- help="List available templates")- parser.add_option("-e", "--edit", dest="edit", action="store_true",- help="Edit spider after creating it")- parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",- help="Dump template to standard output")- parser.add_option("-t", "--template", dest="template", default="basic",- help="Uses a custom template.")- parser.add_option("--force", dest="force", action="store_true",- help="If the spider already exists, overwrite it with the template")+ parser.add_argument("-l", "--list", dest="list", action="store_true",+ help="List available templates")+ parser.add_argument("-e", "--edit", dest="edit", action="store_true",+ help="Edit spider after creating it")+ parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",+ help="Dump template to standard output")+ parser.add_argument("-t", "--template", dest="template", default="basic",+ help="Uses a custom template.")+ parser.add_argument("--force", dest="force", action="store_true",+ help="If the spider already exists, overwrite it with the template")def run(self, args, opts):if opts.list:diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 52118db1b..a3f6b96f4 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):def add_options(self, parser):BaseRunSpiderCommand.add_options(self, parser)- parser.add_option("--spider", dest="spider", default=None,- help="use this spider without looking for one")- parser.add_option("--pipelines", action="store_true",- help="process items through pipelines")- parser.add_option("--nolinks", dest="nolinks", action="store_true",- help="don't show links to follow (extracted requests)")- parser.add_option("--noitems", dest="noitems", action="store_true",- help="don't show scraped items")- parser.add_option("--nocolour", dest="nocolour", action="store_true",- help="avoid using pygments to colorize the output")- parser.add_option("-r", "--rules", dest="rules", action="store_true",- help="use CrawlSpider rules to discover the callback")- parser.add_option("-c", "--callback", dest="callback",- help="use this callback for parsing, instead looking for a callback")- parser.add_option("-m", "--meta", dest="meta",- help="inject extra meta into the Request, it must be a valid raw json string")- parser.add_option("--cbkwargs", dest="cbkwargs",- help="inject extra callback kwargs into the Request, it must be a valid raw json string")- parser.add_option("-d", "--depth", dest="depth", type="int", default=1,- help="maximum depth for parsing requests [default: %default]")- parser.add_option("-v", "--verbose", dest="verbose", action="store_true",- help="print each depth level one by one")+ parser.add_argument("--spider", dest="spider", default=None,+ help="use this spider without looking for one")+ parser.add_argument("--pipelines", action="store_true",+ help="process items through pipelines")+ parser.add_argument("--nolinks", dest="nolinks", action="store_true",+ help="don't show links to follow (extracted requests)")+ parser.add_argument("--noitems", dest="noitems", action="store_true",+ help="don't show scraped items")+ parser.add_argument("--nocolour", dest="nocolour", action="store_true",+ help="avoid using pygments to colorize the output")+ parser.add_argument("-r", "--rules", dest="rules", action="store_true",+ help="use CrawlSpider rules to discover the callback")+ parser.add_argument("-c", "--callback", dest="callback",+ help="use this callback for parsing, instead looking for a callback")+ parser.add_argument("-m", "--meta", dest="meta",+ help="inject extra meta into the Request, it must be a valid raw json string")+ parser.add_argument("--cbkwargs", dest="cbkwargs",+ help="inject extra callback kwargs into the Request, it must be a valid raw json string")+ parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,+ help="maximum depth for parsing requests [default: %default]")+ parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",+ help="print each depth level one by one")@propertydef max_level(self):diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex 8d49e440f..1b2e2601e 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -18,16 +18,16 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("--get", dest="get", metavar="SETTING",- help="print raw setting value")- parser.add_option("--getbool", dest="getbool", metavar="SETTING",- help="print setting value, interpreted as a boolean")- parser.add_option("--getint", dest="getint", metavar="SETTING",- help="print setting value, interpreted as an integer")- parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",- help="print setting value, interpreted as a float")- parser.add_option("--getlist", dest="getlist", metavar="SETTING",- help="print setting value, interpreted as a list")+ parser.add_argument("--get", dest="get", metavar="SETTING",+ help="print raw setting value")+ parser.add_argument("--getbool", dest="getbool", metavar="SETTING",+ help="print setting value, interpreted as a boolean")+ parser.add_argument("--getint", dest="getint", metavar="SETTING",+ help="print setting value, interpreted as an integer")+ parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",+ help="print setting value, interpreted as a float")+ parser.add_argument("--getlist", dest="getlist", metavar="SETTING",+ help="print setting value, interpreted as a list")def run(self, args, opts):settings = self.crawler_process.settingsdiff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex de81986d8..f67a5886a 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -33,12 +33,12 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("-c", dest="code",- help="evaluate the code in the shell, print the result and exit")- parser.add_option("--spider", dest="spider",- help="use this spider")- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,- help="do not handle HTTP 3xx status codes and print response as-is")+ parser.add_argument("-c", dest="code",+ help="evaluate the code in the shell, print the result and exit")+ parser.add_argument("--spider", dest="spider",+ help="use this spider")+ parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,+ help="do not handle HTTP 3xx status codes and print response as-is")def update_vars(self, vars):"""You can use this function to update the Scrapy objects that will bediff --git a/scrapy/commands/version.py b/scrapy/commands/version.pyindex 1237610cb..c6a3c273a 100644--- a/scrapy/commands/version.py+++ b/scrapy/commands/version.py@@ -16,8 +16,8 @@ class Command(ScrapyCommand):def add_options(self, parser):ScrapyCommand.add_options(self, parser)- parser.add_option("--verbose", "-v", dest="verbose", action="store_true",- help="also display twisted/python/platform info (useful for bug reports)")+ parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",+ help="also display twisted/python/platform info (useful for bug reports)")def run(self, args, opts):if opts.verbose:diff --git a/scrapy/commands/view.py b/scrapy/commands/view.pyindex c8f873334..b1f52abe2 100644--- a/scrapy/commands/view.py+++ b/scrapy/commands/view.py@@ -1,3 +1,4 @@+import argparsefrom scrapy.commands import fetchfrom scrapy.utils.response import open_in_browser@@ -12,7 +13,7 @@ class Command(fetch.Command):def add_options(self, parser):super().add_options(parser)- parser.remove_option("--headers")+ parser.add_argument('--headers', help=argparse.SUPPRESS)def _print_response(self, response, opts):open_in_browser(response)diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex ed3848d88..f21ee971d 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -1,6 +1,9 @@import os+import argparsefrom os.path import join, abspath, isfile, existsfrom twisted.internet import defer+from scrapy.commands import parse+from scrapy.settings import Settingsfrom scrapy.utils.testsite import SiteTestfrom scrapy.utils.testproc import ProcessTestfrom scrapy.utils.python import to_unicode@@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}content = '[\n{},\n{"foo": "bar"}\n]'with open(file_path, 'r') as f:self.assertEqual(f.read(), content)++ def test_parse_add_options(self):+ command = parse.Command()+ command.settings = Settings()+ parser = argparse.ArgumentParser(+ prog='scrapy', formatter_class=argparse.HelpFormatter,+ conflict_handler='resolve', prefix_chars='-'+ )+ command.add_options(parser)+ namespace = parser.parse_args(+ ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]+ )+ self.assertTrue(namespace.nolinks)+ self.assertEqual(namespace.depth, 2)+ self.assertEqual(namespace.spider, self.spider_name)+ self.assertTrue(namespace.verbose)diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 7473b53df..7cd19b29a 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -1,6 +1,6 @@import inspectimport json-import optparse+import argparseimport osimport platformimport re@@ -23,7 +23,7 @@ from twisted.python.versions import Versionfrom twisted.trial import unittestimport scrapy-from scrapy.commands import ScrapyCommand+from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatterfrom scrapy.commands.startproject import IGNOREfrom scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode@@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):def setUp(self):self.command = ScrapyCommand()self.command.settings = Settings()- self.parser = optparse.OptionParser(- formatter=optparse.TitledHelpFormatter(),- conflict_handler='resolve',- )+ self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,+ conflict_handler='resolve')self.command.add_options(self.parser)def test_settings_json_string(self):feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'- opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])+ opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])self.command.process_options(args, opts)self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))+ def test_help_formatter(self):+ formatter = ScrapyHelpFormatter(prog='scrapy')+ part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',+ '\n', 'optional arguments:\n', '\n', 'Global Options:\n']+ self.assertEqual(+ formatter._join_parts(part_strings),+ ('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'+ 'Optional Arguments\n==================\n\n'+ 'Global Options\n--------------\n')+ )+class ProjectTest(unittest.TestCase):project_name = 'testproject'@@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):self.assertNotIn('Unhandled Error', log)+class ViewCommandTest(CommandTest):++ def test_methods(self):+ command = view.Command()+ command.settings = Settings()+ parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',+ formatter_class=ScrapyHelpFormatter,+ conflict_handler='resolve')+ command.add_options(parser)+ self.assertEqual(command.short_desc(),+ "Open URL in browser, as seen by Scrapy")+ self.assertIn("URL using the Scrapy downloader and show its",+ command.long_desc())++class CrawlCommandTest(CommandTest):def crawl(self, code, args=()):