Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Simplify skipping uvloop tests.

Andrey Rakhmatullin583df9f7d063
diff --git a/conftest.py b/conftest.py
index e1d4b1213..fa5193470 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,10 @@
+import platform
+import sys
from pathlib import Path
import pytest
+from twisted import version as twisted_version
+from twisted.python.versions import Version
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
@@ -73,6 +77,18 @@ def only_not_asyncio(request, reactor_pytest):
pytest.skip("This test is only run without --reactor=asyncio")
[email protected](autouse=True)
+def requires_uvloop(request):
+ if not request.node.get_closest_marker("requires_uvloop"):
+ return
+ if sys.implementation.name == "pypy":
+ pytest.skip("uvloop does not support pypy properly")
+ if platform.system() == "Windows":
+ pytest.skip("uvloop does not support Windows")
+ if twisted_version == Version("twisted", 21, 2, 0):
+ pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
+
+
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/pytest.ini b/pytest.ini
index 866f0c950..16983be5e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,6 +20,7 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+ requires_uvloop: marks tests as only enabled when uvloop is known to be working
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 014f50e92..03d768d1a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union
from unittest import skipIf
from pytest import mark
-from twisted import version as twisted_version
-from twisted.python.versions import Version
from twisted.trial import unittest
import scrapy
@@ -802,17 +800,7 @@ class MySpider(scrapy.Spider):
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log
)
- @mark.skipif(
- sys.implementation.name == "pypy",
- reason="uvloop does not support pypy properly",
- )
- @mark.skipif(
- platform.system() == "Windows", reason="uvloop does not support Windows"
- )
- @mark.skipif(
- twisted_version == Version("twisted", 21, 2, 0),
- reason="https://twistedmatrix.com/trac/ticket/10106",
- )
+ @mark.requires_uvloop
def test_custom_asyncio_loop_enabled_true(self):
log = self.get_log(
self.debug_log_spider,
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index d54a2cb7e..68e58144b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -8,9 +8,7 @@ from pathlib import Path
from packaging.version import parse as parse_version
from pytest import mark, raises
-from twisted import version as twisted_version
from twisted.internet import defer
-from twisted.python.versions import Version
from twisted.trial import unittest
from w3lib import __version__ as w3lib_version
@@ -466,17 +464,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
log,
)
- @mark.skipif(
- sys.implementation.name == "pypy",
- reason="uvloop does not support pypy properly",
- )
- @mark.skipif(
- platform.system() == "Windows", reason="uvloop does not support Windows"
- )
- @mark.skipif(
- twisted_version == Version("twisted", 21, 2, 0),
- reason="https://twistedmatrix.com/trac/ticket/10106",
- )
+ @mark.requires_uvloop
def test_custom_loop_asyncio(self):
log = self.run_script("asyncio_custom_loop.py")
self.assertIn("Spider closed (finished)", log)
@@ -485,17 +473,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
)
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
- @mark.skipif(
- sys.implementation.name == "pypy",
- reason="uvloop does not support pypy properly",
- )
- @mark.skipif(
- platform.system() == "Windows", reason="uvloop does not support Windows"
- )
- @mark.skipif(
- twisted_version == Version("twisted", 21, 2, 0),
- reason="https://twistedmatrix.com/trac/ticket/10106",
- )
+ @mark.requires_uvloop
def test_custom_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
self.assertIn("Spider closed (finished)", log)
@@ -505,17 +483,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
self.assertIn("async pipeline opened!", log)
- @mark.skipif(
- sys.implementation.name == "pypy",
- reason="uvloop does not support pypy properly",
- )
- @mark.skipif(
- platform.system() == "Windows", reason="uvloop does not support Windows"
- )
- @mark.skipif(
- twisted_version == Version("twisted", 21, 2, 0),
- reason="https://twistedmatrix.com/trac/ticket/10106",
- )
+ @mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
self.assertIn("Spider closed (finished)", log)
@@ -524,17 +492,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
)
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
- @mark.skipif(
- sys.implementation.name == "pypy",
- reason="uvloop does not support pypy properly",
- )
- @mark.skipif(
- platform.system() == "Windows", reason="uvloop does not support Windows"
- )
- @mark.skipif(
- twisted_version == Version("twisted", 21, 2, 0),
- reason="https://twistedmatrix.com/trac/ticket/10106",
- )
+ @mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
self.assertNotIn("Spider closed (finished)", log)