Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Mark slow tests, with their own tox env and CI job

Adrián Chaves9655b0b8eb4b
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 5ff92a571..7562cf22b 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -47,6 +47,9 @@ jobs:
- python-version: "3.11"
env:
TOXENV: botocore
+ - python-version: "3.11"
+ env:
+ TOXENV: slow
- python-version: "3.12.0-rc.2"
env:
@@ -57,6 +60,9 @@ jobs:
- python-version: "3.12.0-rc.2"
env:
TOXENV: extra-deps
+ - python-version: "3.12.0-rc.2"
+ env:
+ TOXENV: slow
steps:
- uses: actions/checkout@v3
diff --git a/pytest.ini b/pytest.ini
index 16983be5e..877fbcd1d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -17,10 +17,12 @@ addopts =
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
+ -m 'not slow'
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
requires_uvloop: marks tests as only enabled when uvloop is known to be working
+ slow: marks tests as slow, not executed by default
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 942584d92..93b9bacaf 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -4,6 +4,8 @@ from pathlib import Path
from time import process_time
from urllib.parse import urlparse
+import pytest
+
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE
@@ -201,36 +203,40 @@ class ResponseUtilsTest(unittest.TestCase):
r5, _openfunc=check_base_url
), "Inject unique base url with conditional comment"
- def test_open_in_browser_redos_comment(self):
- MAX_CPU_TIME = 30
- # Exploit input from
- # https://makenowjust-labs.github.io/recheck/playground/
- # for /<!--.*?-->/ (old pattern to remove comments).
- body = b"-><!--\x00" * (int(DOWNLOAD_MAXSIZE / 7) - 10) + b"->\n<!---->"
+def test_open_in_browser_redos_comment():
+ MAX_CPU_TIME = 30
+
+ # Exploit input from
+ # https://makenowjust-labs.github.io/recheck/playground/
+ # for /<!--.*?-->/ (old pattern to remove comments).
+ body = b"-><!--\x00" * (int(DOWNLOAD_MAXSIZE / 7) - 10) + b"->\n<!---->"
+
+ response = HtmlResponse("https://example.com", body=body)
- response = HtmlResponse("https://example.com", body=body)
+ start_time = process_time()
- start_time = process_time()
+ open_in_browser(response, lambda url: True)
- open_in_browser(response, lambda url: True)
+ end_time = process_time()
+ assert (end_time - start_time) < MAX_CPU_TIME
- end_time = process_time()
- self.assertLess(end_time - start_time, MAX_CPU_TIME)
- def test_open_in_browser_redos_head(self):
- MAX_CPU_TIME = 15
+def test_open_in_browser_redos_head():
+ MAX_CPU_TIME = 15
- # Exploit input from
- # https://makenowjust-labs.github.io/recheck/playground/
- # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
- body = b"<head\t" * int(DOWNLOAD_MAXSIZE / 6)
+ # Exploit input from
+ # https://makenowjust-labs.github.io/recheck/playground/
+ # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
+ body = b"<head\t" * int(DOWNLOAD_MAXSIZE / 6)
- response = HtmlResponse("https://example.com", body=body)
+ response = HtmlResponse("https://example.com", body=body)
- start_time = process_time()
+ start_time = process_time()
- open_in_browser(response, lambda url: True)
+ open_in_browser(response, lambda url: True)
- end_time = process_time()
- self.assertLess(end_time - start_time, MAX_CPU_TIME)
+ end_time = process_time()
+ assert (end_time - start_time) < MAX_CPU_TIME
diff --git a/tox.ini b/tox.ini
index 9c2522a43..be47dec74 100644
--- a/tox.ini
+++ b/tox.ini
@@ -221,3 +221,9 @@ setenv =
{[pinned]setenv}
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+
+[testenv:slow]
+basepython = python3
+commands =
+ {[testenv]commands} -m 'slow'