Scrapy, a fast high-level web crawling & scraping framework for Python.
Mark slow tests, with their own tox env and CI job
Adrián Chaves9655b0b8eb4b
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 5ff92a571..7562cf22b 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -47,6 +47,9 @@ jobs:- python-version: "3.11"env:TOXENV: botocore+ - python-version: "3.11"+ env:+ TOXENV: slow- python-version: "3.12.0-rc.2"env:@@ -57,6 +60,9 @@ jobs:- python-version: "3.12.0-rc.2"env:TOXENV: extra-deps+ - python-version: "3.12.0-rc.2"+ env:+ TOXENV: slowsteps:- uses: actions/checkout@v3diff --git a/pytest.ini b/pytest.iniindex 16983be5e..877fbcd1d 100644--- a/pytest.ini+++ b/pytest.ini@@ -17,10 +17,12 @@ addopts =--ignore=docs/topics/stats.rst--ignore=docs/topics/telnetconsole.rst--ignore=docs/utils+ -m 'not slow'markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passedonly_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passedrequires_uvloop: marks tests as only enabled when uvloop is known to be working+ slow: marks tests as slow, not executed by defaultfilterwarnings =ignore:scrapy.downloadermiddlewares.decompression is deprecatedignore:Module scrapy.utils.reqser is deprecateddiff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 942584d92..93b9bacaf 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -4,6 +4,8 @@ from pathlib import Pathfrom time import process_timefrom urllib.parse import urlparse+import pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Response, TextResponsefrom scrapy.settings.default_settings import DOWNLOAD_MAXSIZE@@ -201,36 +203,40 @@ class ResponseUtilsTest(unittest.TestCase):r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"- def test_open_in_browser_redos_comment(self):- MAX_CPU_TIME = 30- # Exploit input from- # https://makenowjust-labs.github.io/recheck/playground/- # for /<!--.*?-->/ (old pattern to remove comments).- body = b"-><!--\x00" * (int(DOWNLOAD_MAXSIZE / 7) - 10) + b"->\n<!---->"+def test_open_in_browser_redos_comment():+ MAX_CPU_TIME = 30++ # Exploit input from+ # https://makenowjust-labs.github.io/recheck/playground/+ # for /<!--.*?-->/ (old pattern to remove comments).+ body = b"-><!--\x00" * (int(DOWNLOAD_MAXSIZE / 7) - 10) + b"->\n<!---->"++ response = HtmlResponse("https://example.com", body=body)- response = HtmlResponse("https://example.com", body=body)+ start_time = process_time()- start_time = process_time()+ open_in_browser(response, lambda url: True)- open_in_browser(response, lambda url: True)+ end_time = process_time()+ assert (end_time - start_time) < MAX_CPU_TIME- end_time = process_time()- self.assertLess(end_time - start_time, MAX_CPU_TIME)- def test_open_in_browser_redos_head(self):- MAX_CPU_TIME = 15+def test_open_in_browser_redos_head():+ MAX_CPU_TIME = 15- # Exploit input from- # https://makenowjust-labs.github.io/recheck/playground/- # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).- body = b"<head\t" * int(DOWNLOAD_MAXSIZE / 6)+ # Exploit input from+ # https://makenowjust-labs.github.io/recheck/playground/+ # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).+ body = b"<head\t" * int(DOWNLOAD_MAXSIZE / 6)- response = HtmlResponse("https://example.com", body=body)+ response = HtmlResponse("https://example.com", body=body)- start_time = process_time()+ start_time = process_time()- open_in_browser(response, lambda url: True)+ open_in_browser(response, lambda url: True)- end_time = process_time()- self.assertLess(end_time - start_time, MAX_CPU_TIME)+ end_time = process_time()+ assert (end_time - start_time) < MAX_CPU_TIMEdiff --git a/tox.ini b/tox.iniindex 9c2522a43..be47dec74 100644--- a/tox.ini+++ b/tox.ini@@ -221,3 +221,9 @@ setenv ={[pinned]setenv}commands =pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}+++[testenv:slow]+basepython = python3+commands =+ {[testenv]commands} -m 'slow'