Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

3.7 KB
Raw Blame
from __future__ import annotations

from ipaddress import ip_address

from scrapy import Request
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.utils.response import response_from_dict


class CustomResponse(TextResponse):
    attributes: tuple[str, ...] = (*TextResponse.attributes, "custom")

    def __init__(self, *args, custom: str | None = None, **kwargs):
        self.custom = custom
        super().__init__(*args, **kwargs)


class DynamicResponse(Response):
    """Response subclass that extends
    :attr:`~scrapy.http.Response.attributes` on instances, as plugins that
    support several Scrapy versions do."""

    def __init__(self, *args, custom: str | None = None, **kwargs):
        self.custom = custom
        super().__init__(*args, **kwargs)
        self.attributes = (*self.attributes, "custom")


def test_basic() -> None:
    """The class of plain responses is not stored, it is guessed back from the
    response data."""
    response = Response("https://example.com", body=b"\x00\x01")
    assert "_class" not in response.to_dict()
    response2 = response_from_dict(response.to_dict())
    assert response2.__class__ is Response
    assert response2.url == response.url


def test_all_attributes() -> None:
    response = HtmlResponse(
        url="https://example.com",
        status=201,
        headers={"Content-Type": "text/html; charset=latin-1"},
        body=b"\xa3",
        flags=["testFlag"],
        encoding="latin-1",
        ip_address=ip_address("127.0.0.1"),
        protocol="h2",
    )
    response2 = response_from_dict(response.to_dict())
    assert response2.__class__ is HtmlResponse
    for attribute in HtmlResponse.attributes:
        if attribute in {"request", "certificate"}:
            continue
        assert getattr(response2, attribute) == getattr(response, attribute)


def test_headers() -> None:
    response = Response(
        "https://example.com",
        headers=[("Set-Cookie", "a=1"), ("set-cookie", "b=2"), ("etag", "x")],
    )
    headers = response.to_dict()["headers"]
    assert type(headers) is dict
    assert headers == {b"Set-Cookie": [b"a=1", b"b=2"], b"etag": [b"x"]}
    headers[b"etag"].append(b"y")
    assert response.headers.getlist("etag") == [b"x"]
    response2 = response_from_dict(response.to_dict())
    assert response2.headers == response.headers


def test_custom_attributes() -> None:
    response = CustomResponse("https://example.com", custom="value")
    response2 = response_from_dict(response.to_dict())
    assert isinstance(response2, CustomResponse)
    assert response2.custom == "value"


def test_custom_instance_attributes() -> None:
    response = DynamicResponse("https://example.com", custom="value")
    response2 = response_from_dict(response.to_dict())
    assert isinstance(response2, DynamicResponse)
    assert response2.custom == "value"


def test_crawl_attributes_left_out() -> None:
    response = Response(
        "https://example.com",
        request=Request("https://example.com"),
        certificate=object(),
    )
    d = response.to_dict()
    assert "request" not in d
    assert "certificate" not in d
    response2 = response_from_dict(d)
    assert response2.request is None
    assert response2.certificate is None


def test_unknown_class() -> None:
    """Dicts that do not indicate a response class, e.g. cache entries written
    by older Scrapy versions, get a class based on their data."""
    response2 = response_from_dict(
        {
            "url": "https://example.com",
            "status": 200,
            "headers": {b"Content-Type": [b"text/plain"]},
            "body": b"foo",
        }
    )
    assert response2.__class__ is TextResponse
    assert response2.text == "foo"