Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve commands (#7376)

Albert Eduardovich N.5b3761361809
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 8aadc90e7..c43aa626d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -108,17 +108,24 @@ def _print_header(settings: BaseSettings, inproject: bool) -> None:
def _print_commands(settings: BaseSettings, inproject: bool) -> None:
_print_header(settings, inproject)
- print("Usage:")
- print(" scrapy <command> [options] [args]\n")
- print("Available commands:")
+ print(
+ "Usage:\n",
+ " scrapy <command> [options] [args]\n",
+ "Available commands:\n",
+ )
cmds = _get_commands_dict(settings, inproject)
- for cmdname, cmdclass in sorted(cmds.items()):
- print(f" {cmdname:<13} {cmdclass.short_desc()}")
+ print(
+ "\n".join(
+ f" {cmdname:<13} {cmdclass.short_desc()}"
+ for cmdname, cmdclass in sorted(cmds.items())
+ )
+ )
if not inproject:
- print()
- print(" [ more ] More commands available when run from project directory")
- print()
- print('Use "scrapy <command> -h" to see more info about a command')
+ print(
+ "\n",
+ " [ more ] More commands available when run from project directory",
+ )
+ print("\n", 'Use "scrapy <command> -h" to see more info about a command')
def _print_unknown_command_msg(
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index e213d0d94..19b6f6681 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -229,7 +229,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
headings = [
i for i in range(len(part_strings)) if part_strings[i].endswith(":\n")
]
- for index in headings[::-1]:
+ for index in reversed(headings):
char = "-" if "Global Options" in part_strings[index] else "="
part_strings[index] = part_strings[index][:-2].title()
underline = "".join(["\n", (char * len(part_strings[index])), "\n"])
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 1e4e09135..229e8a487 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -102,16 +102,18 @@ class Command(ScrapyCommand):
# start checks
if opts.list:
- for spider, methods in sorted(contract_reqs.items()):
- if not methods and not opts.verbose:
- continue
- print(spider)
- for method in sorted(methods):
- print(f" * {method}")
+ print(
+ "\n".join(
+ f"{spider}\n"
+ + "\n".join(f" * {method}" for method in sorted(methods))
+ for spider, methods in sorted(contract_reqs.items())
+ if methods or opts.verbose
+ )
+ )
else:
- start_time = time.time()
+ start_time = time.monotonic()
self.crawler_process.start()
- stop = time.time()
+ stop = time.monotonic()
result.printErrors()
result.printSummary(start_time, stop)
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index b030be4de..17bcf19b0 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -173,15 +173,21 @@ class Command(ScrapyCommand):
template_file = Path(self.templates_dir, f"{template}.tmpl")
if template_file.exists():
return template_file
- print(f"Unable to find template: {template}\n")
- print('Use "scrapy genspider --list" to see all available templates.')
+ print(
+ f"Unable to find template: {template}\n",
+ 'Use "scrapy genspider --list" to see all available templates.',
+ )
return None
def _list_templates(self) -> None:
- print("Available templates:")
- for file in sorted(Path(self.templates_dir).iterdir()):
- if file.suffix == ".tmpl":
- print(f" {file.stem}")
+ print(
+ "Available templates:\n",
+ "\n".join(
+ f" {file.stem}"
+ for file in sorted(Path(self.templates_dir).iterdir())
+ if file.suffix == ".tmpl"
+ ),
+ )
def _spider_exists(self, name: str) -> bool:
assert self.settings is not None
@@ -200,8 +206,10 @@ class Command(ScrapyCommand):
pass
else:
# if spider with same name exists
- print(f"Spider {name!r} already exists in module:")
- print(f" {spidercls.__module__}")
+ print(
+ f"Spider {name!r} already exists in module:\n",
+ f" {spidercls.__module__}",
+ )
return True
# a file with the same name exists in the target directory
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index ad55dba66..a037f5c9b 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -20,5 +20,4 @@ class Command(ScrapyCommand):
def run(self, args: list[str], opts: argparse.Namespace) -> None:
assert self.settings is not None
spider_loader = get_spider_loader(self.settings)
- for s in sorted(spider_loader.list()):
- print(s)
+ print("\n".join(sorted(spider_loader.list())))
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index c56a7319a..5661a0a48 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -123,12 +123,12 @@ class Command(ScrapyCommand):
)
print(
f"New Scrapy project '{project_name}', using template directory "
- f"'{self.templates_dir}', created in:"
+ f"'{self.templates_dir}', created in:\n",
+ f" {project_dir.resolve()}\n\n",
+ "You can start your first spider with:\n",
+ f" cd {project_dir}\n",
+ " scrapy genspider example example.com",
)
- print(f" {project_dir.resolve()}\n")
- print("You can start your first spider with:")
- print(f" cd {project_dir}")
- print(" scrapy genspider example example.com")
@property
def templates_dir(self) -> str:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 1e91aa0b0..edb03da1b 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -378,27 +378,28 @@ class TestViewCommand:
class TestHelpMessage(TestProjectBase):
- COMMANDS = [
- "parse",
- "startproject",
- "view",
- "crawl",
- "edit",
- "list",
- "fetch",
- "settings",
- "shell",
- "runspider",
- "version",
- "genspider",
- "check",
- "bench",
- ]
-
- def test_help_messages(self, proj_path: Path) -> None:
- for command in self.COMMANDS:
- _, out, _ = proc(command, "-h", cwd=proj_path)
- assert "Usage" in out
+ @pytest.mark.parametrize(
+ "command",
+ [
+ "parse",
+ "startproject",
+ "view",
+ "crawl",
+ "edit",
+ "list",
+ "fetch",
+ "settings",
+ "shell",
+ "runspider",
+ "version",
+ "genspider",
+ "check",
+ "bench",
+ ],
+ )
+ def test_help_messages(self, proj_path: Path, command: str) -> None:
+ _, out, _ = proc(command, "-h", cwd=proj_path)
+ assert "Usage" in out
class TestPopCommandName: