Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

5.1 KB
Raw Blame
from typing import Any

import pytest

from scrapy.http import (
    Headers,
    HtmlResponse,
    JsonResponse,
    Response,
    TextResponse,
    XmlResponse,
)
from scrapy.responsetypes import responsetypes


@pytest.mark.parametrize(
    ("source", "cls"),
    [
        ("data.bin", Response),
        ("file.txt", TextResponse),
        ("file.xml.gz", Response),
        ("file.xml", XmlResponse),
        ("file.html", HtmlResponse),
        ("file.unknownext", Response),
    ],
)
def test_from_filename(source: str, cls: type[Response]) -> None:
    assert responsetypes.from_filename(source) is cls


@pytest.mark.parametrize(
    ("source", "cls"),
    [
        pytest.param(b'attachment; filename="data.xml"', XmlResponse, id="quoted"),
        pytest.param(b"attachment; filename=data.xml", XmlResponse, id="unquoted"),
        pytest.param("attachment;filename=data£.tar.gz".encode(), Response, id="utf-8"),
        pytest.param(
            "attachment;filename=dataµ.tar.gz".encode("latin-1"),
            Response,
            id="latin-1",
        ),
        pytest.param(
            "attachment;filename=data高.doc".encode("gbk"), Response, id="gbk"
        ),
        pytest.param(
            "attachment;filename=دورهdata.html".encode("cp720"),
            HtmlResponse,
            id="cp720",
        ),
        pytest.param(
            "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
            XmlResponse,
            id="iso2022_jp",
        ),
    ],
)
def test_from_content_disposition(source: bytes, cls: type[Response]) -> None:
    assert responsetypes.from_content_disposition(source) is cls


def test_from_content_disposition_no_filename() -> None:
    assert responsetypes.from_content_disposition(b"attachment") is Response


@pytest.mark.parametrize(
    ("source", "cls"),
    [
        ("text/html; charset=UTF-8", HtmlResponse),
        ("text/xml; charset=UTF-8", XmlResponse),
        ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
        ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
        ("application/xml; charset=UTF-8", XmlResponse),
        ("application/octet-stream", Response),
        ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
        ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
        ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
        (b"application/x-download; filename=\x80dummy.txt", Response),
    ],
)
def test_from_content_type(source: str | bytes, cls: type[Response]) -> None:
    assert responsetypes.from_content_type(source) is cls


@pytest.mark.parametrize(
    ("source", "cls"),
    [
        (b"\x03\x02\xdf\xdd\x23", Response),
        (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
        (b"<html><head><title>Hello</title></head>", HtmlResponse),
        # https://codersblock.com/blog/the-smallest-valid-html5-page/
        (b"<!DOCTYPE html>\n<title>.</title>", HtmlResponse),
        (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
    ],
)
def test_from_body(source: bytes, cls: type[Response]) -> None:
    assert responsetypes.from_body(source) is cls


@pytest.mark.parametrize(
    ("source", "cls"),
    [
        pytest.param(
            {"Content-Type": ["text/html; charset=utf-8"]},
            HtmlResponse,
            id="content-type",
        ),
        pytest.param(
            {
                "Content-Type": ["text/html; charset=utf-8"],
                "Content-Encoding": ["gzip"],
            },
            Response,
            id="content-encoding",
        ),
        pytest.param(
            {
                "Content-Type": ["application/octet-stream"],
                "Content-Disposition": ["attachment; filename=data.txt"],
            },
            TextResponse,
            id="content-disposition",
        ),
    ],
)
def test_from_headers(source: dict[str, list[str]], cls: type[Response]) -> None:
    assert responsetypes.from_headers(Headers(source)) is cls


# TODO: add more tests that check precedence between the different arguments
@pytest.mark.parametrize(
    ("source", "cls"),
    [
        pytest.param(
            {"url": "http://www.example.com/data.csv"}, TextResponse, id="url"
        ),
        # headers takes precedence over url
        pytest.param(
            {
                "headers": Headers({"Content-Type": ["text/html; charset=utf-8"]}),
                "url": "http://www.example.com/item/",
            },
            HtmlResponse,
            id="content-type-and-url",
        ),
        pytest.param(
            {
                "headers": Headers(
                    {"Content-Disposition": ['attachment; filename="data.xml.gz"']}
                ),
                "url": "http://www.example.com/page/",
            },
            Response,
            id="content-disposition-and-url",
        ),
    ],
)
def test_from_args(source: dict[str, Any], cls: type[Response]) -> None:
    assert responsetypes.from_args(**source) is cls


def test_custom_mime_types_loaded() -> None:
    # check that mime.types files shipped with scrapy are loaded
    assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test"