Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

allowing to run .pyw files (#4646)

Akshay Sharma159e2b2e2fbb
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index aedd8c2ce..b957c29fb 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -11,7 +11,7 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
- if fext != '.py':
+ if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2899e5f24..3e54a0948 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest):
class RunSpiderCommandTest(CommandTest):
+ spider_filename = 'myspider.py'
+
debug_log_spider = """
import scrapy
@@ -507,11 +509,23 @@ class MySpider(scrapy.Spider):
return []
"""
+ badspider = """
+import scrapy
+
+class BadSpider(scrapy.Spider):
+ name = "bad"
+ def start_requests(self):
+ raise Exception("oops!")
+ """
+
@contextmanager
- def _create_file(self, content, name):
+ def _create_file(self, content, name=None):
tmpdir = self.mktemp()
os.mkdir(tmpdir)
- fname = abspath(join(tmpdir, name))
+ if name:
+ fname = abspath(join(tmpdir, name))
+ else:
+ fname = abspath(join(tmpdir, self.spider_filename))
with open(fname, 'w') as f:
f.write(content)
try:
@@ -519,12 +533,12 @@ class MySpider(scrapy.Spider):
finally:
rmtree(tmpdir)
- def runspider(self, code, name='myspider.py', args=()):
+ def runspider(self, code, name=None, args=()):
with self._create_file(code, name) as fname:
return self.proc('runspider', fname, *args)
- def get_log(self, code, name='myspider.py', args=()):
- p, stdout, stderr = self.runspider(code, name=name, args=args)
+ def get_log(self, code, name=None, args=()):
+ p, stdout, stderr = self.runspider(code, name, args=args)
return stderr
def test_runspider(self):
@@ -556,7 +570,7 @@ class MySpider(scrapy.Spider):
# which is intended,
# but this should not be because of DNS lookup error
# assumption: localhost will resolve in all cases (true?)
- log = self.get_log("""
+ dnscache_spider = """
import scrapy
class MySpider(scrapy.Spider):
@@ -565,23 +579,20 @@ class MySpider(scrapy.Spider):
def parse(self, response):
return {'test': 'value'}
-""",
- args=('-s', 'DNSCACHE_ENABLED=False'))
- print(log)
+"""
+ log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
self.assertNotIn("DNSLookupError", log)
self.assertIn("INFO: Spider opened", log)
def test_runspider_log_short_names(self):
log1 = self.get_log(self.debug_log_spider,
args=('-s', 'LOG_SHORT_NAMES=1'))
- print(log1)
self.assertIn("[myspider] DEBUG: It Works!", log1)
self.assertIn("[scrapy]", log1)
self.assertNotIn("[scrapy.core.engine]", log1)
log2 = self.get_log(self.debug_log_spider,
args=('-s', 'LOG_SHORT_NAMES=0'))
- print(log2)
self.assertIn("[myspider] DEBUG: It Works!", log2)
self.assertNotIn("[scrapy]", log2)
self.assertIn("[scrapy.core.engine]", log2)
@@ -599,15 +610,7 @@ class MySpider(scrapy.Spider):
self.assertIn('Unable to load', log)
def test_start_requests_errors(self):
- log = self.get_log("""
-import scrapy
-
-class BadSpider(scrapy.Spider):
- name = "bad"
- def start_requests(self):
- raise Exception("oops!")
- """, name="badspider.py")
- print(log)
+ log = self.get_log(self.badspider, name='badspider.py')
self.assertIn("start_requests", log)
self.assertIn("badspider.py", log)
@@ -696,6 +699,54 @@ class MySpider(scrapy.Spider):
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
+
+ spider_filename = 'myspider.pyw'
+
+ def setUp(self):
+ super(WindowsRunSpiderCommandTest, self).setUp()
+
+ def test_start_requests_errors(self):
+ log = self.get_log(self.badspider, name='badspider.pyw')
+ self.assertIn("start_requests", log)
+ self.assertIn("badspider.pyw", log)
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_run_good_spider(self):
+ super().test_run_good_spider()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_runspider(self):
+ super().test_runspider()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_runspider_dnscache_disabled(self):
+ super().test_runspider_dnscache_disabled()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_runspider_log_level(self):
+ super().test_runspider_log_level()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_runspider_log_short_names(self):
+ super().test_runspider_log_short_names()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_runspider_no_spider_found(self):
+ super().test_runspider_no_spider_found()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_output(self):
+ super().test_output()
+
+ @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+ def test_overwrite_output(self):
+ super().test_overwrite_output()
+
+ def test_runspider_unable_to_load(self):
+ raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
+
+
class BenchCommandTest(CommandTest):
def test_run(self):