Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

9.9 KB
Raw Blame
import copy
from abc import ABCMeta
from typing import Any
from unittest import mock

import pytest

from scrapy.item import Field, Item, ItemMeta


class TestItem:
    def test_simple(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        i["name"] = "name"
        assert i["name"] == "name"

    def test_init(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        with pytest.raises(KeyError):
            i["name"]

        i2 = TestItem(name="john doe")
        assert i2["name"] == "john doe"

        i3 = TestItem({"name": "john doe"})
        assert i3["name"] == "john doe"

        i4 = TestItem(i3)
        assert i4["name"] == "john doe"

        with pytest.raises(KeyError):
            TestItem({"name": "john doe", "other": "foo"})

    def test_invalid_field(self):
        class TestItem(Item):
            pass

        i = TestItem()
        with pytest.raises(KeyError):
            i["field"] = "text"
        with pytest.raises(KeyError):
            i["field"]

    def test_delitem(self):
        class TestItem(Item):
            name = Field()

        i = TestItem(name="John")
        del i["name"]
        with pytest.raises(KeyError):
            i["name"]

        with pytest.raises(KeyError):
            del i["name"]

    def test_repr(self):
        class TestItem(Item):
            name = Field()
            number = Field()

        i = TestItem()
        i["name"] = "John Doe"
        i["number"] = 123
        itemrepr = repr(i)

        assert itemrepr == "{'name': 'John Doe', 'number': 123}"

        i2 = eval(itemrepr)
        assert i2["name"] == "John Doe"
        assert i2["number"] == 123

    def test_private_attr(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        i._private = "test"
        assert i._private == "test"

    def test_raise_getattr(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        with pytest.raises(AttributeError):
            i.name

    def test_raise_setattr(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        with pytest.raises(AttributeError):
            i.name = "john"  # type: ignore[assignment]

    def test_custom_methods(self):
        class TestItem(Item):
            name = Field()

            def get_name(self) -> Any:
                return self["name"]

            def change_name(self, name: str) -> None:
                self["name"] = name

        i = TestItem()
        with pytest.raises(KeyError):
            i.get_name()
        i["name"] = "lala"
        assert i.get_name() == "lala"
        i.change_name("other")
        assert i.get_name() == "other"

    def test_metaclass(self):
        class TestItem(Item):
            name = Field()
            keys = Field()  # type: ignore[assignment]
            values = Field()  # type: ignore[assignment]

        i = TestItem()
        i["name"] = "John"
        assert list(i.keys()) == ["name"]  # type: ignore[operator]
        assert list(i.values()) == ["John"]  # type: ignore[operator]

        i["keys"] = "Keys"
        i["values"] = "Values"
        assert sorted(i.keys()) == ["keys", "name", "values"]  # type: ignore[operator]
        assert sorted(i.values()) == ["John", "Keys", "Values"]  # type: ignore[operator]

    def test_metaclass_with_fields_attribute(self):
        class TestItem(Item):
            fields = {"new": Field(default="X")}

        item = TestItem(new="New")
        assert list(item.keys()) == ["new"]
        assert list(item.values()) == ["New"]

    def test_fields_order(self):
        class TestItem(Item):
            name = Field()
            keys = Field()  # type: ignore[assignment]
            values = Field()  # type: ignore[assignment]

        assert list(TestItem.fields) == ["name", "keys", "values"]

    def test_fields_order_inheritance(self):
        class ParentItem(Item):
            name = Field()
            keys = Field()  # type: ignore[assignment]
            values = Field()  # type: ignore[assignment]

        class TestItem(ParentItem):
            extra = Field()
            keys = Field(serializer=str)

        # Inherited fields come first, in their definition order, followed by
        # the fields newly defined in the subclass. A redefined field keeps the
        # position of its first definition while taking the new metadata.
        assert list(TestItem.fields) == ["name", "keys", "values", "extra"]
        assert TestItem.fields["keys"] == {"serializer": str}

    def test_metaclass_inheritance(self):
        class ParentItem(Item):
            name = Field()
            keys = Field()  # type: ignore[assignment]
            values = Field()  # type: ignore[assignment]

        class TestItem(ParentItem):
            keys = Field()

        i = TestItem()
        i["keys"] = 3
        assert list(i.keys()) == ["keys"]  # type: ignore[operator]
        assert list(i.values()) == [3]  # type: ignore[operator]

    def test_metaclass_multiple_inheritance_simple(self):
        class A(Item):
            fields = {"load": Field(default="A")}
            save = Field(default="A")

        class B(A):
            pass

        class C(Item):
            fields = {"load": Field(default="C")}
            save = Field(default="C")

        class D(B, C):
            pass

        item = D(save="X", load="Y")
        assert item["save"] == "X"
        assert item["load"] == "Y"
        assert D.fields == {"load": {"default": "A"}, "save": {"default": "A"}}

        # D class inverted
        class E(C, B):
            pass

        assert E(save="X")["save"] == "X"
        assert E(load="X")["load"] == "X"
        assert E.fields == {"load": {"default": "C"}, "save": {"default": "C"}}

    def test_metaclass_multiple_inheritance_diamond(self):
        class A(Item):
            fields = {"update": Field(default="A")}
            save = Field(default="A")
            load = Field(default="A")

        class B(A):
            pass

        class C(A):
            fields = {"update": Field(default="C")}
            save = Field(default="C")

        class D(B, C):
            fields = {"update": Field(default="D")}
            load = Field(default="D")

        assert D(save="X")["save"] == "X"
        assert D(load="X")["load"] == "X"
        assert D.fields == {
            "save": {"default": "C"},
            "load": {"default": "D"},
            "update": {"default": "D"},
        }

        # D class inverted
        class E(C, B):
            load = Field(default="E")

        assert E(save="X")["save"] == "X"
        assert E(load="X")["load"] == "X"
        assert E.fields == {
            "save": {"default": "C"},
            "load": {"default": "E"},
            "update": {"default": "C"},
        }

    def test_metaclass_multiple_inheritance_without_metaclass(self):
        class A(Item):
            fields = {"load": Field(default="A")}
            save = Field(default="A")

        class B(A):
            pass

        class C:
            fields = {"load": Field(default="C")}
            not_allowed = Field(default="not_allowed")
            save = Field(default="C")

        class D(B, C):
            pass

        with pytest.raises(KeyError):
            D(not_allowed="value")
        assert D(save="X")["save"] == "X"
        assert D.fields == {"save": {"default": "A"}, "load": {"default": "A"}}

        # D class inverted
        class E(C, B):
            pass

        with pytest.raises(KeyError):
            E(not_allowed="value")
        assert E(save="X")["save"] == "X"
        assert E.fields == {"save": {"default": "A"}, "load": {"default": "A"}}

    def test_to_dict(self):
        class TestItem(Item):
            name = Field()

        i = TestItem()
        i["name"] = "John"
        assert dict(i) == {"name": "John"}

    def test_copy(self):
        class TestItem(Item):
            name = Field()

        item = TestItem({"name": "lower"})
        copied_item = item.copy()
        assert id(item) != id(copied_item)
        copied_item["name"] = copied_item["name"].upper()
        assert item["name"] != copied_item["name"]

    def test_copy_copy(self):
        class TestItem(Item):
            name = Field()

        item = TestItem({"name": "lower"})
        copied_item = copy.copy(item)
        copied_item["name"] = copied_item["name"].upper()
        assert item["name"] != copied_item["name"]

    def test_deepcopy(self):
        class TestItem(Item):
            tags = Field()

        item = TestItem({"tags": ["tag1"]})
        copied_item = item.deepcopy()
        item["tags"].append("tag2")
        assert item["tags"] != copied_item["tags"]


class TestItemMeta:
    def test_new_method_propagates_classcell(self):
        new_mock = mock.Mock(side_effect=ABCMeta.__new__)
        base = ItemMeta.__bases__[0]

        with mock.patch.object(base, "__new__", new_mock):

            class MyItem(Item):
                def f(self):
                    # For rationale of this see:
                    # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
                    return __class__  # type: ignore[name-defined]

            MyItem()

        (first_call, second_call) = new_mock.call_args_list[-2:]

        *_, attrs = first_call[0]
        assert "__classcell__" not in attrs
        *_, attrs = second_call[0]
        assert "__classcell__" in attrs


class TestItemMetaClassCellRegression:
    def test_item_meta_classcell_regression(self):
        class MyItem(Item, metaclass=ItemMeta):
            def __init__(self, *args, **kwargs):  # pylint: disable=useless-parent-delegation
                # This call to super() trigger the __classcell__ propagation
                # requirement. When not done properly raises an error:
                # TypeError: __class__ set to <class '__main__.MyItem'>
                # defining 'MyItem' as <class '__main__.MyItem'>
                super().__init__(*args, **kwargs)