Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import codecs
import gc
import weakref
from typing import cast
from unittest import mock
import frostwork
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, TextResponse, XmlResponse
from scrapy.http.response.text import MetaRefresh
from scrapy.selector import Selector
from tests.utils.bases.http_response import TestResponseBase
class TestTextResponse(TestResponseBase):
response_class = TextResponse
def _links_response(self) -> TextResponse:
return cast("TextResponse", super()._links_response())
def _links_response_no_href(self) -> TextResponse:
return cast("TextResponse", super()._links_response_no_href())
def test_follow_None_encoding(self):
# unlike the base Response, TextResponse.follow() falls back to the
# response encoding when encoding is None instead of raising
r = self.response_class("http://example.com", body=b"hello", encoding="cp1252")
req = r.follow("foo", encoding=None)
assert req.encoding == "cp1252"
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
"http://www.example.com", body="hello", encoding="cp866"
)
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
assert isinstance(r2, self.response_class)
assert r2.url == "http://www.example.com/other"
assert r2.encoding == "cp866"
assert r3.url == "http://www.example.com/other"
assert r3._declared_encoding() == "latin1"
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class("http://www.example.com/")
self._assert_response_encoding(resp, "utf-8")
# make sure urls are converted to str
resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
assert isinstance(resp.url, str)
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="utf-8"
)
assert resp.url == b"http://www.example.com/price/\xc2\xa3".decode()
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="latin-1"
)
assert resp.url == "http://www.example.com/price/\xa3"
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
assert resp.url == b"http://www.example.com/price/\xc2\xa3".decode()
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
assert resp.url == "http://www.example.com/price/\xa3"
def test_unicode_body(self):
unicode_string = (
"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
"\u0442\u0435\u043a\u0441\u0442"
)
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="unicode body")
original_string = unicode_string.encode("cp1251")
r1 = self.response_class(
"http://www.example.com", body=original_string, encoding="cp1251"
)
# check response.text
assert isinstance(r1.text, str)
assert r1.text == unicode_string
@pytest.mark.parametrize(
("body", "encoding", "text", "reused"),
[
("\xa3\U0001f600", "utf8", "\xa3\U0001f600", True),
("\ufeffWORD", "utf-8", "WORD", False),
("\x80", "latin-1", "\u20ac", False),
],
)
def test_str_body_text(
self, body: str, encoding: str, text: str, reused: bool
) -> None:
r = self.response_class("http://www.example.com", body=body, encoding=encoding)
assert r.text == text
assert (r.text is body) is reused
def test_encoding(self):
r1 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
r2 = self.response_class(
"http://www.example.com", encoding="utf-8", body="\xa3"
)
r3 = self.response_class(
"http://www.example.com",
body=b"\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
r5 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=None"]},
)
r6 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gb2312"]},
)
r7 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gbk"]},
)
r8 = self.response_class(
"http://www.example.com",
body=codecs.BOM_UTF8 + b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=cp1251"]},
)
r9 = self.response_class(
"http://www.example.com",
body=b"\x80",
headers={
"Content-type": [b"application/x-download; filename=\x80dummy.txt"]
},
)
assert r1._headers_encoding() == "utf-8"
assert r2._headers_encoding() is None
assert r2._declared_encoding() == "utf-8"
self._assert_response_encoding(r2, "utf-8")
assert r3._headers_encoding() == "cp1252"
assert r3._declared_encoding() == "cp1252"
assert r4._headers_encoding() is None
assert r5._headers_encoding() is None
assert r8._headers_encoding() == "cp1251"
assert r9._headers_encoding() is None
assert r8._declared_encoding() == "utf-8"
assert r9._declared_encoding() is None
self._assert_response_encoding(r5, "utf-8")
self._assert_response_encoding(r8, "utf-8")
assert r4._body_inferred_encoding() is not None
assert r4._body_inferred_encoding() != "ascii"
assert r9._body_inferred_encoding() is not None
assert r9._body_inferred_encoding() != "ascii"
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")
self._assert_response_values(r6, "gbk", "\u2015")
self._assert_response_values(r7, "gbk", "\u2015")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
r = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=UNKNOWN"]},
body=b"\xc2\xa3",
)
assert r._declared_encoding() is None
self._assert_response_values(r, "utf-8", "\xa3")
@pytest.mark.parametrize(
("url", "encoding"),
[("http://www.example.com", "cp932"), ("http://www.example.cn", "gbk")],
)
def test_body_inferred_encoding_of_undeclared_legacy_page(
self, url: str, encoding: str
) -> None:
body = "ICD10 国際疾病分類第10版2013年版 病名マスター".encode("shift-jis")
r = self.response_class(url, body=body)
assert r._declared_encoding() is None
assert r.encoding == encoding
assert r.text == body.decode(r.encoding)
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
body = b"\xff\xfeh\x00i\x00"
r = self.response_class(
"http://www.example.com",
body=body,
encoding="utf-16",
)
assert r.encoding == "utf-16"
assert r.text == "hi"
@pytest.mark.parametrize(
("encoding", "body", "text"),
[
("cp437", "<p>é", "<p>é"),
("hz", b"~{Dc:C~}", "你好"),
("foo", b"\xc2\xa3", "£"),
("cp437", codecs.BOM_UTF8 + b"\xc2\xa3", "£"),
],
)
def test_non_browser_encoding(
self, encoding: str, body: str | bytes, text: str
) -> None:
with pytest.warns(ScrapyDeprecationWarning, match=repr(encoding)):
r = self.response_class(
"http://www.example.com", body=body, encoding=encoding
)
assert r.encoding == encoding
assert r.text == text
def test_replace_replacement_encoding(self) -> None:
r = self.response_class(
"http://www.example.com",
headers={"Content-Type": ["text/html; charset=hz"]},
body=b"~{Dc:C~}",
)
assert r.encoding == "replacement"
assert r.replace(url="http://www.example.com/other").text == "�"
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=utf-8"]},
body=b"\xef\xbb\xbfWORD\xe3\xab",
)
assert r6.encoding == "utf-8"
assert r6.text == "WORD\ufffd"
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
headers = {"Content-type": ["text/html; charset=utf-8"]}
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
# Body caching sideeffect isn't triggered when encoding is declared in
# content-type header but BOM still need to be removed from decoded
# body
response = self.response_class(url, headers=headers, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, headers=headers, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
def test_replace_wrong_encoding(self):
"""Test invalid chars are replaced properly"""
r = self.response_class(
"http://www.example.com",
encoding="utf-8",
body=b"PREFIX\xe3\xabSUFFIX",
)
assert r.text == "PREFIX\ufffdSUFFIX"
# Do not destroy html tags due to encoding bugs
r = self.response_class(
"http://example.com",
encoding="utf-8",
body=b"\xf0<span>value</span>",
)
assert "<span>value</span>" in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "html"
# property is cached
assert response.selector is response.selector # pylint: disable=comparison-with-itself
assert response.selector.response is response
assert response.selector.xpath("//title/text()").getall() == ["Some page"]
assert response.selector.css("title::text").getall() == ["Some page"]
assert response.selector.re("Some (.*)</title>") == ["page"]
def test_selector_response_refcounting(self):
response = self.response_class("http://www.example.com", body=b"<a>b</a>")
selector = response.selector
ref = weakref.ref(response)
gc.disable()
try:
del response
assert ref() is None
finally:
gc.enable()
assert selector.response is None
def test_selector_shortcuts(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//title/text()").getall()
== response.selector.xpath("//title/text()").getall()
)
assert (
response.css("title::text").getall()
== response.selector.css("title::text").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b'<html><head><title>Some page</title><body><p class="content">A nice paragraph.</p></body></html>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"normalize-space(//p[@class=$pclass])", pclass="content"
).getall()
== response.xpath('normalize-space(//p[@class="content"])').getall()
)
assert (
response.xpath(
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content",
pcount=1,
).getall()
== response.xpath(
'//title[count(following::p[@class="content"])=1]/text()'
).getall()
)
def test_urljoin_with_base_url(self):
body = b'<html><body><base href="https://example.net"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"/test"
)
absolute = "https://example.net/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere/"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/elsewhere/test"
assert joined == absolute
def test_follow_redirect_ignores_base_url(self):
"""The Location header is resolved against the response URL, not against
the base URL of the document, which only applies to its own links."""
body = b'<html><body><base href="https://example.net"></body></html>'
response = self.response_class(
"http://www.example.com/dir/index",
body=body,
headers={"Location": "/test"},
)
assert response.urljoin("/test") == "https://example.net/test"
assert response.follow_redirect().url == "http://www.example.com/test"
def test_base_url_serialized(self) -> None:
response = self.response_class("http://www.EXAMPLE.com", body=b"")
assert response.base_url == "http://www.example.com/"
@pytest.mark.parametrize(
("body", "base_url"),
[
(b"", "http://www.example.com/a/b"),
(b'<base href="c/">', "http://www.example.com/a/c/"),
(
b'<base target="_blank"><base href="/c"><base href="/d">',
"http://www.example.com/c",
),
],
)
def test_base_url(self, body: bytes, base_url: str) -> None:
response = self.response_class("http://www.example.com/a/b", body=body)
assert response.base_url == base_url
@pytest.mark.parametrize(
("body", "meta_refresh"),
[
(b"", None),
(
b'<base href="/c/"><meta http-equiv="refresh" content="5;url=d">',
MetaRefresh(5.0, "http://www.example.com/c/d"),
),
(
(
b'<meta http-equiv="refresh" content="2;url=/a">'
b'<meta http-equiv="refresh" content="1;url=/b">'
b'<meta http-equiv="refresh" content="1;url=/c">'
b'<meta http-equiv="refresh" content="3;url=/d">'
),
MetaRefresh(1.0, "http://www.example.com/c"),
),
(
(
b'<noscript><meta http-equiv="refresh" content="0;url=/a">'
b"</noscript><template>"
b'<meta http-equiv="refresh" content="0;url=/b"></template>'
b'<script>document.write(\'<meta http-equiv="refresh" '
b'content="0;url=/c">\')</script>'
),
None,
),
],
)
def test_meta_refresh(self, body: bytes, meta_refresh: MetaRefresh | None) -> None:
response = self.response_class("http://www.example.com", body=body)
assert response.meta_refresh == meta_refresh
def test_html_metadata_once(self) -> None:
body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'
response = self.response_class("http://www.example.com", body=body)
with mock.patch("frostwork.navigation", wraps=frostwork.navigation) as read:
assert response.base_url == "http://www.example.com/a/"
assert response.meta_refresh == MetaRefresh(
0.0, "http://www.example.com/a/b"
)
read.assert_called_once()
def test_html_metadata_set(self) -> None:
body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'
response = self.response_class("http://www.example.com", body=body)
response.base_url = "http://example.net/"
assert response.meta_refresh == MetaRefresh(0.0, "http://www.example.com/a/b")
assert response.base_url == "http://example.net/"
response.meta_refresh = None
assert response.meta_refresh is None
response = self.response_class("http://www.example.com", body=body)
response.meta_refresh = (1.0, "http://example.net/")
assert response.base_url == "http://www.example.com/a/"
assert response.meta_refresh == MetaRefresh(1.0, "http://example.net/")
meta_refresh = MetaRefresh(2.0, "http://example.org/")
response.meta_refresh = meta_refresh
assert response.meta_refresh is meta_refresh
def test_follow_selector(self):
resp = self._links_response()
urls = [
"http://example.com/sample2.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html#foo",
"http://www.google.com/something",
"http://example.com/innertag.html",
"http://example.com/page%204.html",
]
# select <a> elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# select <link> elements
self._assert_followed_url(
Selector(text='<link href="foo"></link>').css("link")[0],
"http://example.com/foo",
response=resp,
)
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
with pytest.raises(
ValueError, match="Only <a> and <link> elements are supported"
):
resp.follow(resp.css("div")[0])
def test_follow_selector_list(self):
resp = self._links_response()
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a")) # type: ignore[arg-type]
def test_follow_selector_invalid(self):
resp = self._links_response()
with pytest.raises(ValueError, match="Unsupported"):
resp.follow(resp.xpath("count(//div)")[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
for src in resp.css("img::attr(src)"):
self._assert_followed_url(src, "http://example.com/sample2.jpg")
def test_follow_selector_no_href(self):
resp = self.response_class(
url="http://example.com",
body=b"<html><body><a name=123>click me</a></body></html>",
)
with pytest.raises(ValueError, match="no href"):
resp.follow(resp.css("a")[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
"http://example.com",
body=b"""<html><body><a href=" foo\n">click me</a></body></html>""",
)
self._assert_followed_url(
resp.css("a")[0], "http://example.com/foo", response=resp
)
self._assert_followed_url(
resp.css("a::attr(href)")[0],
"http://example.com/foo",
response=resp,
)
def test_follow_encoding(self):
resp1 = self.response_class(
"http://example.com",
encoding="utf8",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
)
self._assert_followed_url(
resp1.css("a")[0],
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
response=resp1,
encoding="utf8",
)
resp2 = self.response_class(
"http://example.com",
encoding="cp1251",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
"cp1251"
),
)
self._assert_followed_url(
resp2.css("a")[0],
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
response=resp2,
encoding="cp1251",
)
def test_follow_all_css(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
assert expected == extracted
def test_follow_all_css_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
assert expected == extracted1
extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
assert expected == extracted2
def test_follow_all_xpath(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
assert expected == [r.url for r in extracted]
def test_follow_all_xpath_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [
r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
]
assert expected == extracted1
extracted2 = [
r.url
for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
]
assert expected == extracted2
def test_follow_all_too_many_arguments(self):
response = self._links_response()
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
response.follow_all(
css='a[href*="example.com"]',
xpath='//a[contains(@href, "example.com")]',
)
def test_json_response(self):
json_body = b"""{"ip": "109.187.217.200"}"""
json_response = self.response_class("http://www.example.com", body=json_body)
assert json_response.json() == {"ip": "109.187.217.200"}
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with pytest.raises(
ValueError, match=r"(Expecting value|Unexpected '<'): line 1"
):
text_response.json()
def test_json_response_non_utf8(self):
response = self.response_class(
"http://www.example.com",
body='{"message": "café"}'.encode("cp1252"),
headers={"Content-Type": "application/json"},
)
assert response.json() == {"message": "café"}
def test_json_response_wrong_charset(self):
response = self.response_class(
"http://www.example.com",
body='{"message": "café"}'.encode(),
headers={"Content-Type": "application/json; charset=iso-8859-1"},
)
assert response.json() == {"message": "café"}
@pytest.mark.parametrize(
"json_body", [b"""{"ip": "109.187.217.200"}""", b"""null"""]
)
def test_cache_json_response(self, json_body: bytes) -> None:
json_response = self.response_class("http://www.example.com", body=json_body)
with mock.patch("json.loads") as mock_json:
for _ in range(2):
json_response.json()
mock_json.assert_called_once_with(json_body)
class TestHtmlResponse(TestTextResponse):
response_class = HtmlResponse
def test_html_encoding(self):
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head><body>Price: \xa3100</body></html>'
"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "iso-8859-1", body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
Price: \xa3100
"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# for conflicting declarations headers must take precedence
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head><body>Price: \xa3100</body></html>'
"""
r3 = self.response_class(
"http://www.example.com",
body=body,
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
self._assert_response_values(r3, "iso-8859-1", body)
# make sure replace() preserves the encoding of the original response
body = b"New body \xa3"
r4 = r3.replace(body=body)
self._assert_response_values(r4, "iso-8859-1", body)
def test_html5_meta_charset(self):
body = b"""<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "gb2312", body)
class TestXmlResponse(TestTextResponse):
response_class = XmlResponse
def test_xml_encoding(self):
body = b"<xml></xml>"
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "utf-8", body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# make sure replace() preserves the explicit encoding passed in the __init__ method
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
body2 = b"New body"
r4 = r3.replace(body=body2)
self._assert_response_values(r4, "utf-8", body2)
def test_replace_encoding(self):
# make sure replace() keeps the previous encoding unless overridden explicitly
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
body2 = b"""<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
r5 = self.response_class("http://www.example.com", body=body)
r6 = r5.replace(body=body2)
r7 = r5.replace(body=body2, encoding="utf-8")
self._assert_response_values(r5, "iso-8859-1", body)
self._assert_response_values(r6, "iso-8859-1", body2)
self._assert_response_values(r7, "utf-8", body2)
def test_selector(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "xml"
# property is cached
assert response.selector is response.selector # pylint: disable=comparison-with-itself
assert response.selector.response is response
assert response.selector.xpath("//elem/text()").getall() == ["value"]
def test_selector_shortcuts(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//elem/text()").getall()
== response.selector.xpath("//elem/text()").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b"""<?xml version="1.0" encoding="utf-8"?>
<xml xmlns:somens="http://scrapy.org">
<somens:elem>value</somens:elem>
</xml>"""
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
== response.selector.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
)
response.selector.register_namespace("s2", "http://scrapy.org")
assert (
response.xpath(
"//s1:elem/text()", namespaces={"s1": "http://scrapy.org"}
).getall()
== response.selector.xpath("//s2:elem/text()").getall()
)
class CustomResponse(TextResponse):
attributes = (*TextResponse.attributes, "foo", "bar")
def __init__(self, *args, **kwargs) -> None:
self.foo = kwargs.pop("foo", None)
self.bar = kwargs.pop("bar", None)
self.lost = kwargs.pop("lost", None)
super().__init__(*args, **kwargs)
class TestCustomResponse(TestTextResponse):
response_class = CustomResponse
def test_copy(self) -> None:
super().test_copy()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.copy()
assert isinstance(r2, self.response_class)
assert r1.foo == r2.foo
assert r1.bar == r2.bar
assert r1.lost == "lost"
assert r2.lost is None
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
assert isinstance(r2, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r2.foo == "new-foo"
assert r2.bar == "new-bar"
assert r2.lost == "new-lost"
r3 = r1.replace(foo="new-foo", bar="new-bar")
assert isinstance(r3, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r3.foo == "new-foo"
assert r3.bar == "new-bar"
assert r3.lost is None
r4 = r1.replace(foo="new-foo")
assert isinstance(r4, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r4.foo == "new-foo"
assert r4.bar == "bar"
assert r4.lost is None
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'unknown'",
):
r1.replace(unknown="unknown")