Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import copy
from collections.abc import Callable, Iterator, Mapping, MutableMapping
from operator import methodcaller
from typing import Any
import pytest
from scrapy.http import Request
from scrapy.utils._datatypes import (
CaseInsensitiveDict,
LocalCache,
LocalWeakReferencedCache,
SequenceExclude,
)
class TestCaseInsensitiveDict:
dict_class = CaseInsensitiveDict
def test_init_dict(self):
seq = {"red": 1, "black": 3}
d = self.dict_class(seq)
assert d["red"] == 1
assert d["black"] == 3
def test_init_pair_sequence(self):
seq = (("red", 1), ("black", 3))
d = self.dict_class(seq)
assert d["red"] == 1
assert d["black"] == 3
def test_init_mapping(self):
class MyMapping(Mapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key: str) -> int:
return self._d[key]
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self) -> int:
return len(self._d)
seq = MyMapping(red=1, black=3)
d = self.dict_class(seq)
assert d["red"] == 1
assert d["black"] == 3
def test_init_mutable_mapping(self):
class MyMutableMapping(MutableMapping[str, int]):
def __init__(self, **kwargs: int) -> None:
self._d = kwargs
def __getitem__(self, key: str) -> int:
return self._d[key]
def __setitem__(self, key: str, value: int) -> None:
self._d[key] = value
def __delitem__(self, key: str) -> None:
del self._d[key]
def __iter__(self) -> Iterator[str]:
return iter(self._d)
def __len__(self) -> int:
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
d = self.dict_class(seq)
assert d["red"] == 1
assert d["black"] == 3
def test_caseless(self):
d = self.dict_class()
d["key_Lower"] = 1
assert d["KEy_loWer"] == 1
assert d.get("KEy_loWer") == 1
d["KEY_LOWER"] = 3
assert d["key_Lower"] == 3
assert d.get("key_Lower") == 3
def test_delete(self):
d = self.dict_class({"key_lower": 1})
del d["key_LOWER"]
with pytest.raises(KeyError):
d["key_LOWER"]
with pytest.raises(KeyError):
d["key_lower"]
def test_getdefault(self):
d = self.dict_class()
assert d.get("c", 5) == 5
d["c"] = 10
assert d.get("c", 5) == 10
def test_setdefault(self):
d = self.dict_class({"a": 1, "b": 2})
r = d.setdefault("A", 5)
assert r == 1
assert d["A"] == 1
r = d.setdefault("c", 5)
assert r == 5
assert d["C"] == 5
def test_fromkeys(self):
keys = ("a", "b")
d = self.dict_class.fromkeys(keys)
assert d["A"] is None
assert d["B"] is None
d = self.dict_class.fromkeys(keys, 1)
assert d["A"] == 1
assert d["B"] == 1
instance = self.dict_class()
d = instance.fromkeys(keys)
assert d["A"] is None
assert d["B"] is None
d = instance.fromkeys(keys, 1)
assert d["A"] == 1
assert d["B"] == 1
def test_contains(self):
d = self.dict_class()
d["a"] = 1
assert "A" in d
def test_pop(self):
d = self.dict_class()
d["a"] = 1
assert d.pop("A") == 1
with pytest.raises(KeyError):
d.pop("A")
def test_normkey(self):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normkey(self, key):
return key.title()
d = MyDict()
d["key-one"] = 2
assert list(d.keys()) == ["Key-One"]
def test_normvalue(self):
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
if value is not None:
return value + 1
return None
d = MyDict({"key": 1})
assert d["key"] == 2
assert d.get("key") == 2
d = MyDict()
d["key"] = 1
assert d["key"] == 2
assert d.get("key") == 2
d = MyDict()
d.setdefault("key", 1)
assert d["key"] == 2
assert d.get("key") == 2
d = MyDict()
d.update({"key": 1})
assert d["key"] == 2
assert d.get("key") == 2
d = MyDict.fromkeys(("key",), 1)
assert d["key"] == 2
assert d.get("key") == 2
def test_copy(self):
h1 = self.dict_class({"header1": "value"})
h2 = copy.copy(h1)
assert isinstance(h2, self.dict_class)
assert h1 == h2
assert h1.get("header1") == h2.get("header1")
assert h1.get("header1") == h2.get("HEADER1")
h3 = h1.copy()
assert isinstance(h3, self.dict_class)
assert h1 == h3
assert h1.get("header1") == h3.get("header1")
assert h1.get("header1") == h3.get("HEADER1")
@pytest.mark.parametrize(
"copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]
)
def test_copy_is_independent(
self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]
) -> None:
h1 = self.dict_class({"header1": "value1", "header2": "value2"})
h2 = copier(h1)
del h2["header1"]
h2["header3"] = "value3"
assert "header1" in h1
assert "header3" not in h1
assert dict(h1) == {"header1": "value1", "header2": "value2"}
def test_repr(self):
d1 = self.dict_class({"foo": "bar"})
assert repr(d1) == "<CaseInsensitiveDict: {'foo': 'bar'}>"
d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
assert repr(d2) == "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>"
def test_iter(self):
d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
iterkeys = iter(d)
assert isinstance(iterkeys, Iterator)
assert list(iterkeys) == ["AsDf", "FoO"]
@pytest.mark.parametrize(
"copier", [copy.copy, methodcaller("copy")], ids=["copy.copy", "copy-method"]
)
def test_copy_keeps_values(
self, copier: Callable[[CaseInsensitiveDict], CaseInsensitiveDict]
) -> None:
class MyDict(self.dict_class): # type: ignore[misc,name-defined]
def _normvalue(self, value):
return value + 1
d = MyDict({"key": 1})
assert copier(d)["key"] == 2
def test_ior(self):
d = self.dict_class({"header1": "value1"})
d |= {"HEADER1": "value2", "header2": "value3"}
assert len(d) == 2
assert d["HeAdEr1"] == "value2"
assert d["HeAdEr2"] == "value3"
def test_ior_mapping(self):
d = self.dict_class({"header1": "value1"})
d |= self.dict_class({"HEADER1": "value2"})
assert len(d) == 1
assert d["HeAdEr1"] == "value2"
class TestSequenceExclude:
def test_list(self):
seq = [1, 2, 3]
d = SequenceExclude(seq)
assert 0 in d
assert 4 in d
assert 2 not in d
def test_range(self):
seq = range(10, 20)
d = SequenceExclude(seq)
assert 5 in d
assert 20 in d
assert 15 not in d
def test_range_step(self):
seq = range(10, 20, 3)
d = SequenceExclude(seq)
are_not_in = [v for v in range(10, 20, 3) if v in d]
assert are_not_in == []
are_not_in = [v for v in range(10, 20) if v in d]
assert are_not_in == [11, 12, 14, 15, 17, 18]
def test_string_seq(self):
seq = "cde"
d = SequenceExclude(seq)
chars = "".join(v for v in "abcdefg" if v in d)
assert chars == "abfg"
def test_stringset_seq(self):
seq = set("cde")
d = SequenceExclude(seq)
chars = "".join(v for v in "abcdefg" if v in d)
assert chars == "abfg"
def test_set(self):
"""Anything that is not in the supplied sequence will evaluate as 'in' the container."""
seq = {-3, "test", 1.1}
d = SequenceExclude(seq)
assert 0 in d
assert "foo" in d
assert 3.14 in d
assert set("bar") in d
# supplied sequence is a set, so checking for list (non)inclusion fails
with pytest.raises(TypeError):
["a", "b", "c"] in d # noqa: B015
for v in [-3, "test", 1.1]:
assert v not in d
class TestLocalCache:
def test_cache_with_limit(self):
cache: LocalCache[str, int] = LocalCache(limit=2)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
assert len(cache) == 2
assert "a" not in cache
assert "b" in cache
assert "c" in cache
assert cache["b"] == 2
assert cache["c"] == 3
def test_cache_without_limit(self):
maximum = 10**4
cache: LocalCache[str, int] = LocalCache()
for x in range(maximum):
cache[str(x)] = x
assert len(cache) == maximum
for x in range(maximum):
assert str(x) in cache
assert cache[str(x)] == x
def test_cache_with_zero_limit(self):
cache: LocalCache[str, int] = LocalCache(limit=0)
cache["a"] = 1
cache["b"] = 2
cache["c"] = 3
assert len(cache) == 0
assert "a" not in cache
assert "b" not in cache
assert "c" not in cache
def test_cache_update_existing_key_does_not_evict(self):
# Updating an existing key must not evict another entry or drop the
# cache below its limit (regression: it used to evict the oldest key).
cache: LocalCache[str, int] = LocalCache(limit=2)
cache["a"] = 1
cache["b"] = 2
cache["b"] = 20
assert len(cache) == 2
assert "a" in cache
assert cache["a"] == 1
assert cache["b"] == 20
class TestLocalWeakReferencedCache:
def test_cache_with_limit(self):
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
limit=2
)
r1 = Request("https://example.org")
r2 = Request("https://example.com")
r3 = Request("https://example.net")
cache[r1] = 1
cache[r2] = 2
cache[r3] = 3
assert len(cache) == 2
assert r1 not in cache
assert r2 in cache
assert r3 in cache
assert cache[r1] is None
assert cache[r2] == 2
assert cache[r3] == 3
del r2
assert len(cache) == 1
def test_cache_non_weak_referenceable_objects(self):
cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
k1 = None
k2 = 1
k3 = [1, 2, 3]
cache[k1] = 1
cache[k2] = 2
cache[k3] = 3
assert k1 not in cache
assert k2 not in cache
assert k3 not in cache
assert len(cache) == 0
def test_cache_without_limit(self):
maximum = 10**4
cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
refs = []
for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))
cache[refs[-1]] = x
assert len(cache) == maximum
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i
del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable
# delete half of the objects, make sure that is reflected in the cache
for _ in range(maximum // 2):
refs.pop()
assert len(cache) == maximum // 2
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i