Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix running tests with -n auto, enable it on CI (#7257)

Andrey Rakhmatullin6e0a0e476a09
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 4d199c960..3adf3de38 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -13,6 +13,8 @@ concurrency:
jobs:
tests:
runs-on: macos-latest
+ env:
+ PYTEST_ADDOPTS: -n auto
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 585cdd307..c07cd49f0 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -13,6 +13,8 @@ concurrency:
jobs:
tests:
runs-on: ubuntu-latest
+ env:
+ PYTEST_ADDOPTS: -n auto
strategy:
fail-fast: false
matrix:
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 9b03b594d..48aa56e15 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -13,6 +13,8 @@ concurrency:
jobs:
tests:
runs-on: windows-latest
+ env:
+ PYTEST_ADDOPTS: -n auto
strategy:
fail-fast: false
matrix:
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 6b63dc056..7f5060701 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -700,6 +700,10 @@ class TestFTPFileStore:
meta = {"foo": "bar"}
path = "full/filename"
with MockFTPServer() as ftp_server:
+ # normally set via FilesPipeline.from_crawler()
+ FTPFilesStore.FTP_USERNAME = "anonymous"
+ FTPFilesStore.FTP_PASSWORD = "guest"
+
store = FTPFilesStore(ftp_server.url("/"))
empty_dict = yield store.stat_file(path, info=None)
assert empty_dict == {}
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 8ca588978..4e359fd33 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import re
import warnings
from asyncio import sleep
from typing import Any
@@ -140,11 +141,16 @@ class TestMain:
for item_or_request in super().start_requests():
yield item_or_request
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead"
- ) as messages:
+ msg = "use Spider.start() instead"
+ with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws:
await self._test_spider(TestSpider, [])
- assert messages[0].filename.endswith("test_spider_start.py")
+
+ for w in ws:
+ if isinstance(w.message, ScrapyDeprecationWarning) and msg in str(
+ w.message
+ ):
+ assert w.filename.endswith("test_spider_start.py")
+ break
async def _test_start(self, start_, expected_items=None):
class TestSpider(Spider):
diff --git a/tox.ini b/tox.ini
index 3be564502..213ecbef6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,6 +26,7 @@ deps =
{[test-requirements]deps}
pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502
passenv =
+ PYTEST_ADDOPTS
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY