Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix merge issues (#8276)

Andrey Rakhmatullin24de02bebcd0
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 4eefa058f..3a8b05203 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -258,28 +258,6 @@ class MySpider(scrapy.Spider):
assert "(Pdb)" not in out
-class TestSettingsCommand:
- @pytest.mark.parametrize(
- ("option", "setting", "expected"),
- [
- ("--get", "BOT_NAME", "scrapybot"),
- ("--getbool", "COOKIES_ENABLED", "True"),
- ("--getint", "CONCURRENT_REQUESTS", "16"),
- ("--getfloat", "DOWNLOAD_DELAY", "0.0"),
- ("--getlist", "SPIDER_MODULES", "[]"),
- ],
- )
- def test_get(self, option: str, setting: str, expected: str) -> None:
- returncode, out, err = proc("settings", option, setting)
- assert returncode == 0, err
- assert out.startswith(expected)
-
- def test_no_option(self) -> None:
- returncode, out, err = proc("settings")
- assert returncode == 0, err
- assert not out
-
-
class TestCommandCrawlerProcess(TestProjectBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
@@ -846,6 +824,26 @@ class MySpider(scrapy.Spider):
assert returncode == 2
assert "Unable to find spider: nope" in err
+ @pytest.mark.parametrize(
+ ("option", "setting", "expected"),
+ [
+ ("--get", "BOT_NAME", "scrapybot"),
+ ("--getbool", "COOKIES_ENABLED", "True"),
+ ("--getint", "CONCURRENT_REQUESTS", "16"),
+ ("--getfloat", "DOWNLOAD_DELAY", "0.0"),
+ ("--getlist", "SPIDER_MODULES", "[]"),
+ ],
+ )
+ def test_get_standalone(self, option: str, setting: str, expected: str) -> None:
+ returncode, out, err = proc("settings", option, setting)
+ assert returncode == 0, err
+ assert out.startswith(expected)
+
+ def test_no_option(self) -> None:
+ returncode, out, err = proc("settings")
+ assert returncode == 0, err
+ assert not out
+
class TestBenchCommand:
@pytest.mark.parametrize("use_reactor", [True, False])
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 6f5f0d80e..87bf71070 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -110,7 +110,7 @@ _EMPTY_DOMAIN_DROPPED = _Limitation(
dropped=True,
)
_DOTLESS_DOMAIN_DROPPED = _Limitation(
- "Cookies with a Domain attribute without dots are dropped (#6410)",
+ "Cookies with a Domain attribute without dots are dropped",
dropped=True,
)
@@ -255,7 +255,7 @@ def _cookie_domain_params() -> Iterable[ParameterSet]:
)
-_LOCAL_HOST_DROPPED = "Cookies for localhost and IP-address hosts are dropped (#6410)"
+_IPV6_HOST_DROPPED = "Cookies for IPv6 hosts are dropped"
def _local_host_params() -> Iterable[ParameterSet]:
@@ -264,9 +264,14 @@ def _local_host_params() -> Iterable[ParameterSet]:
("ipv4", "127.0.0.1", "127.0.0.1"),
("ipv6", "[::1]", "::1"),
):
+ known: str | None
for cookie_domain in (None, host_domain):
- # Only IPv6 hosts also lose cookies without a Domain attribute.
- known = cookie_domain is not None or host_id == "ipv6"
+ if host_id == "ipv6":
+ known = _IPV6_HOST_DROPPED
+ elif host_id == "localhost" and cookie_domain is not None:
+ known = _DOTLESS_DOMAIN_DROPPED.reason
+ else:
+ known = None
for source in ("request", "response"):
yield pytest.param(
source,
@@ -274,9 +279,7 @@ def _local_host_params() -> Iterable[ParameterSet]:
cookie_domain,
id=f"{source}-{host_id}-{'self' if cookie_domain else 'unset'}",
marks=(
- [pytest.mark.xfail(strict=True, reason=_LOCAL_HOST_DROPPED)]
- if known
- else []
+ [pytest.mark.xfail(strict=True, reason=known)] if known else []
),
)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 965cec7d4..976436e83 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -883,10 +883,11 @@ class TestHttpCompression:
(b"not compressed", "zstd"),
],
)
- def test_process_response_invalid(self, body, encoding):
+ @coroutine_test
+ async def test_process_response_invalid(self, body: bytes, encoding: str) -> None:
request = Request("http://example.com")
response = Response(
"http://example.com", body=body, headers={"Content-Encoding": encoding}
)
with pytest.raises(DecompressionError):
- self.mw.process_response(request, response)
+ await self.mw.process_response(request, response)