Scrapy, a fast high-level web crawling & scraping framework for Python.
allowing to run .pyw files (#4646)
Akshay Sharma159e2b2e2fbb
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex aedd8c2ce..b957c29fb 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -11,7 +11,7 @@ def _import_file(filepath):abspath = os.path.abspath(filepath)dirname, file = os.path.split(abspath)fname, fext = os.path.splitext(file)- if fext != '.py':+ if fext not in ('.py', '.pyw'):raise ValueError(f"Not a Python source file: {abspath}")if dirname:sys.path = [dirname] + sys.pathdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 2899e5f24..3e54a0948 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest):class RunSpiderCommandTest(CommandTest):+ spider_filename = 'myspider.py'+debug_log_spider = """import scrapy@@ -507,11 +509,23 @@ class MySpider(scrapy.Spider):return []"""+ badspider = """+import scrapy++class BadSpider(scrapy.Spider):+ name = "bad"+ def start_requests(self):+ raise Exception("oops!")+ """+@contextmanager- def _create_file(self, content, name):+ def _create_file(self, content, name=None):tmpdir = self.mktemp()os.mkdir(tmpdir)- fname = abspath(join(tmpdir, name))+ if name:+ fname = abspath(join(tmpdir, name))+ else:+ fname = abspath(join(tmpdir, self.spider_filename))with open(fname, 'w') as f:f.write(content)try:@@ -519,12 +533,12 @@ class MySpider(scrapy.Spider):finally:rmtree(tmpdir)- def runspider(self, code, name='myspider.py', args=()):+ def runspider(self, code, name=None, args=()):with self._create_file(code, name) as fname:return self.proc('runspider', fname, *args)- def get_log(self, code, name='myspider.py', args=()):- p, stdout, stderr = self.runspider(code, name=name, args=args)+ def get_log(self, code, name=None, args=()):+ p, stdout, stderr = self.runspider(code, name, args=args)return stderrdef test_runspider(self):@@ -556,7 +570,7 @@ class MySpider(scrapy.Spider):# which is intended,# but this should not be because of DNS lookup error# assumption: localhost will resolve in all cases (true?)- log = self.get_log("""+ dnscache_spider = """import scrapyclass MySpider(scrapy.Spider):@@ -565,23 +579,20 @@ class MySpider(scrapy.Spider):def parse(self, response):return {'test': 'value'}-""",- args=('-s', 'DNSCACHE_ENABLED=False'))- print(log)+"""+ log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))self.assertNotIn("DNSLookupError", log)self.assertIn("INFO: Spider opened", log)def test_runspider_log_short_names(self):log1 = self.get_log(self.debug_log_spider,args=('-s', 'LOG_SHORT_NAMES=1'))- print(log1)self.assertIn("[myspider] DEBUG: It Works!", log1)self.assertIn("[scrapy]", log1)self.assertNotIn("[scrapy.core.engine]", log1)log2 = self.get_log(self.debug_log_spider,args=('-s', 'LOG_SHORT_NAMES=0'))- print(log2)self.assertIn("[myspider] DEBUG: It Works!", log2)self.assertNotIn("[scrapy]", log2)self.assertIn("[scrapy.core.engine]", log2)@@ -599,15 +610,7 @@ class MySpider(scrapy.Spider):self.assertIn('Unable to load', log)def test_start_requests_errors(self):- log = self.get_log("""-import scrapy--class BadSpider(scrapy.Spider):- name = "bad"- def start_requests(self):- raise Exception("oops!")- """, name="badspider.py")- print(log)+ log = self.get_log(self.badspider, name='badspider.py')self.assertIn("start_requests", log)self.assertIn("badspider.py", log)@@ -696,6 +699,54 @@ class MySpider(scrapy.Spider):self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)+class WindowsRunSpiderCommandTest(RunSpiderCommandTest):++ spider_filename = 'myspider.pyw'++ def setUp(self):+ super(WindowsRunSpiderCommandTest, self).setUp()++ def test_start_requests_errors(self):+ log = self.get_log(self.badspider, name='badspider.pyw')+ self.assertIn("start_requests", log)+ self.assertIn("badspider.pyw", log)++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_run_good_spider(self):+ super().test_run_good_spider()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_runspider(self):+ super().test_runspider()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_runspider_dnscache_disabled(self):+ super().test_runspider_dnscache_disabled()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_runspider_log_level(self):+ super().test_runspider_log_level()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_runspider_log_short_names(self):+ super().test_runspider_log_short_names()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_runspider_no_spider_found(self):+ super().test_runspider_no_spider_found()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_output(self):+ super().test_output()++ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")+ def test_overwrite_output(self):+ super().test_overwrite_output()++ def test_runspider_unable_to_load(self):+ raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")++class BenchCommandTest(CommandTest):def test_run(self):