Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

5.8 KB
Raw Blame
from __future__ import annotations

import os
import sys
from pathlib import Path
from unittest import mock

import pytest

from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Field, Item
from scrapy.utils.misc import (
    arg_to_iter,
    build_from_crawler,
    load_object,
    rel_has_nofollow,
    set_environ,
    walk_modules,
    walk_modules_iter,
)


def test_load_object_class() -> None:
    obj = load_object(Field)
    assert obj is Field
    obj = load_object("scrapy.item.Field")
    assert obj is Field


def test_load_object_function() -> None:
    obj = load_object(load_object)
    assert obj is load_object
    obj = load_object("scrapy.utils.misc.load_object")
    assert obj is load_object


def test_load_object_exceptions() -> None:
    with pytest.raises(ImportError):
        load_object("nomodule999.mod.function")
    with pytest.raises(NameError):
        load_object("scrapy.utils.misc.load_object999")
    with pytest.raises(TypeError):
        load_object({})  # type: ignore[call-overload]


def test_walk_modules() -> None:
    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
    expected = [
        "tests.test_utils_misc.test_walk_modules",
        "tests.test_utils_misc.test_walk_modules.mod",
        "tests.test_utils_misc.test_walk_modules.mod.mod0",
        "tests.test_utils_misc.test_walk_modules.mod1",
    ]
    assert {m.__name__ for m in mods} == set(expected)

    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
    expected = [
        "tests.test_utils_misc.test_walk_modules.mod",
        "tests.test_utils_misc.test_walk_modules.mod.mod0",
    ]
    assert {m.__name__ for m in mods} == set(expected)

    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
    expected = [
        "tests.test_utils_misc.test_walk_modules.mod1",
    ]
    assert {m.__name__ for m in mods} == set(expected)

    with pytest.raises(ImportError):
        for _ in walk_modules_iter("nomodule999"):
            pass
    with (
        pytest.raises(ImportError),
        pytest.warns(
            ScrapyDeprecationWarning,
            match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
            "removed in a future version of Scrapy. "
            "Use scrapy.utils.misc.walk_modules_iter instead.",
        ),
    ):
        walk_modules("nomodule999")


def test_walk_modules_egg() -> None:
    egg = str(Path(__file__).parent / "test.egg")
    sys.path.append(egg)
    try:
        mods = walk_modules_iter("testegg")
        expected = [
            "testegg.spiders",
            "testegg.spiders.a",
            "testegg.spiders.b",
            "testegg",
        ]
        assert {m.__name__ for m in mods} == set(expected)
    finally:
        sys.path.remove(egg)


def test_arg_to_iter() -> None:
    class TestItem(Item):
        name = Field()

    assert hasattr(arg_to_iter(None), "__iter__")
    assert hasattr(arg_to_iter(100), "__iter__")
    assert hasattr(arg_to_iter("lala"), "__iter__")
    assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
    assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")

    assert not list(arg_to_iter(None))
    assert list(arg_to_iter("lala")) == ["lala"]
    assert list(arg_to_iter(100)) == [100]
    assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
    assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
    assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
    assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]


def test_build_from_crawler() -> None:
    crawler = mock.MagicMock(spec_set=["settings"])
    args = (True, 100.0)
    kwargs = {"key": "val"}

    def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
        build_from_crawler(mock, crawler, *args, **kwargs)
        if hasattr(mock, "from_crawler"):
            mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
            assert mock.call_count == 0
        else:
            mock.assert_called_once_with(*args, **kwargs)

    # Check usage of correct constructor using 2 mocks:
    #   1. with no alternative constructors
    #   2. with from_crawler() constructor
    spec_sets = (
        ["__qualname__"],
        ["__qualname__", "from_crawler"],
    )
    for specs in spec_sets:
        m = mock.MagicMock(spec_set=specs)
        _test_with_crawler(m, crawler)
        m.reset_mock()

    # Check adoption of crawler
    m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
    m.from_crawler.return_value = None
    with pytest.raises(TypeError):
        build_from_crawler(m, crawler, *args, **kwargs)


def test_set_environ(monkeypatch: pytest.MonkeyPatch) -> None:
    assert os.environ.get("some_test_environ") is None
    with set_environ(some_test_environ="test_value"):
        assert os.environ.get("some_test_environ") == "test_value"
    assert os.environ.get("some_test_environ") is None

    monkeypatch.setenv("some_test_environ", "test")
    assert os.environ.get("some_test_environ") == "test"
    with set_environ(some_test_environ="test_value"):
        assert os.environ.get("some_test_environ") == "test_value"
    assert os.environ.get("some_test_environ") == "test"


def test_rel_has_nofollow() -> None:
    assert rel_has_nofollow("ugc nofollow") is True
    assert rel_has_nofollow("ugc,nofollow") is True
    assert rel_has_nofollow("ugc") is False
    assert rel_has_nofollow("nofollow") is True
    assert rel_has_nofollow("nofollowfoo") is False
    assert rel_has_nofollow("foonofollow") is False
    assert rel_has_nofollow("ugc,  ,  nofollow") is True
    # rel attribute values are ASCII case-insensitive per the HTML spec
    assert rel_has_nofollow("NoFollow") is True
    assert rel_has_nofollow("NOFOLLOW") is True
    assert rel_has_nofollow("UGC NoFollow") is True
    assert rel_has_nofollow("ugc,NoFollow") is True