Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

10.8 KB
Raw Blame
import re
from typing import Any

import pytest

from scrapy import Request, Spider
from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler

UNSET = object()


@pytest.mark.parametrize(
    ("allowed_domain", "url", "allowed"),
    [
        ("example.com", "http://example.com/1", True),
        ("example.com", "http://example.org/1", False),
        ("example.com", "http://sub.example.com/1", True),
        ("sub.example.com", "http://sub.example.com/1", True),
        ("sub.example.com", "http://example.com/1", False),
        ("example.com", "http://example.com:8000/1", True),
        ("example.com", "http://example.org/example.com", False),
        ("example.com", "http://example.org/foo.example.com", False),
        ("example.com", "http://example.com.example", False),
        ("a.example", "http://nota.example", False),
        ("b.a.example", "http://notb.a.example", False),
    ],
)
def test_process_request_domain_filtering(allowed_domain, url, allowed):
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request(url)
    if allowed:
        assert mw.process_request(request) is None
    else:
        with pytest.raises(IgnoreRequest):
            mw.process_request(request)


@pytest.mark.parametrize(
    ("value", "filtered"),
    [
        (UNSET, True),
        (None, True),
        (False, True),
        (True, False),
    ],
)
def test_process_request_dont_filter(value, filtered):
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    kwargs: dict[str, Any] = {}
    if value is not UNSET:
        kwargs["dont_filter"] = value
    request = Request("https://b.example", **kwargs)
    if filtered:
        with pytest.raises(IgnoreRequest):
            mw.process_request(request)
    else:
        assert mw.process_request(request) is None


@pytest.mark.parametrize(
    ("allow_offsite", "dont_filter", "filtered"),
    [
        (True, UNSET, False),
        (True, None, False),
        (True, False, False),
        (True, True, False),
        (False, UNSET, True),
        (False, None, True),
        (False, False, True),
        (False, True, False),
    ],
)
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    kwargs: dict[str, Any] = {"meta": {}}
    if allow_offsite is not UNSET:
        kwargs["meta"]["allow_offsite"] = allow_offsite
    if dont_filter is not UNSET:
        kwargs["dont_filter"] = dont_filter
    request = Request("https://b.example", **kwargs)
    if filtered:
        with pytest.raises(IgnoreRequest):
            mw.process_request(request)
    else:
        assert mw.process_request(request) is None


@pytest.mark.parametrize(
    "value",
    [
        UNSET,
        None,
        [],
    ],
)
def test_process_request_no_allowed_domains(value):
    crawler = get_crawler(Spider)
    kwargs: dict[str, Any] = {}
    if value is not UNSET:
        kwargs["allowed_domains"] = value
    crawler.spider = crawler._create_spider(name="a", **kwargs)
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request("https://example.com")
    assert mw.process_request(request) is None


def test_process_request_invalid_domains():
    crawler = get_crawler(Spider)
    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
    crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request("https://a.example")
    assert mw.process_request(request) is None
    for letter in ("b", "c"):
        request = Request(f"https://{letter}.example")
        with pytest.raises(IgnoreRequest):
            mw.process_request(request)


@pytest.mark.parametrize(
    ("allowed_domain", "url", "allowed"),
    [
        ("example.com", "http://example.com/1", True),
        ("example.com", "http://example.org/1", False),
        ("example.com", "http://sub.example.com/1", True),
        ("sub.example.com", "http://sub.example.com/1", True),
        ("sub.example.com", "http://example.com/1", False),
        ("example.com", "http://example.com:8000/1", True),
        ("example.com", "http://example.org/example.com", False),
        ("example.com", "http://example.org/foo.example.com", False),
        ("example.com", "http://example.com.example", False),
        ("a.example", "http://nota.example", False),
        ("b.a.example", "http://notb.a.example", False),
    ],
)
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request(url)
    if allowed:
        mw.request_scheduled(request, crawler.spider)
    else:
        with pytest.raises(IgnoreRequest):
            mw.request_scheduled(request, crawler.spider)


@pytest.mark.parametrize(
    ("value", "filtered"),
    [
        (UNSET, True),
        (None, True),
        (False, True),
        (True, False),
    ],
)
def test_request_scheduled_dont_filter(value, filtered):
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    kwargs: dict[str, Any] = {}
    if value is not UNSET:
        kwargs["dont_filter"] = value
    request = Request("https://b.example", **kwargs)
    if filtered:
        with pytest.raises(IgnoreRequest):
            mw.request_scheduled(request, crawler.spider)
    else:
        mw.request_scheduled(request, crawler.spider)


@pytest.mark.parametrize(
    "value",
    [
        UNSET,
        None,
        [],
    ],
)
def test_request_scheduled_no_allowed_domains(value):
    crawler = get_crawler(Spider)
    kwargs: dict[str, Any] = {}
    if value is not UNSET:
        kwargs["allowed_domains"] = value
    crawler.spider = crawler._create_spider(name="a", **kwargs)
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request("https://example.com")
    mw.request_scheduled(request, crawler.spider)


def test_request_scheduled_invalid_domains():
    crawler = get_crawler(Spider)
    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
    crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request("https://a.example")
    mw.request_scheduled(request, crawler.spider)
    for letter in ("b", "c"):
        request = Request(f"https://{letter}.example")
        with pytest.raises(IgnoreRequest):
            mw.request_scheduled(request, crawler.spider)


def test_repeated_offsite_domain():
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    req1 = Request("http://other.org/1")
    req2 = Request("http://other.org/2")
    with pytest.raises(IgnoreRequest):
        mw.process_request(req1)
    assert "other.org" in mw.domains_seen
    assert crawler.stats.get_value("offsite/domains") == 1
    assert crawler.stats.get_value("offsite/filtered") == 1
    with pytest.raises(IgnoreRequest):
        mw.process_request(req2)
    assert crawler.stats.get_value("offsite/domains") == 1  # not incremented again
    assert crawler.stats.get_value("offsite/filtered") == 2


def test_should_follow_override():
    class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
        def should_follow(self, request: Request, spider: Spider) -> bool:
            allowed_domains: list[str] = getattr(spider, "allowed_domains", [])
            return urlparse_cached(request).hostname in allowed_domains

    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
    mw = build_from_crawler(RootOnlyOffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    assert mw.process_request(Request("https://example.com/1")) is None
    with pytest.raises(IgnoreRequest):
        mw.process_request(Request("https://www.example.com/1"))


def test_ignore_request_reason():
    crawler = get_crawler(Spider)
    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(crawler.spider)
    request = Request("http://other.org/1")
    with pytest.raises(
        IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")
    ):
        mw.process_request(request)


class DomainSpider(Spider):
    name = "a"
    allowed_domains: list[str]


def test_dynamic_allowed_domains():
    crawler = get_crawler(DomainSpider)
    spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
    crawler.spider = spider
    mw = build_from_crawler(OffsiteMiddleware, crawler)
    mw.spider_opened(spider)

    with pytest.raises(IgnoreRequest):
        mw.process_request(Request("https://b.example"))

    spider.allowed_domains.append("b.example")
    assert mw.process_request(Request("https://b.example")) is None

    spider.allowed_domains.remove("a.example")
    with pytest.raises(IgnoreRequest):
        mw.process_request(Request("https://a.example"))


def test_dynamic_allowed_domains_caching():
    calls = 0

    class TrackingMiddleware(OffsiteMiddleware):
        def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
            nonlocal calls
            calls += 1
            return super().get_host_regex(spider)

    crawler = get_crawler(DomainSpider)
    spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
    crawler.spider = spider
    mw = build_from_crawler(TrackingMiddleware, crawler)
    mw.spider_opened(spider)

    for _ in range(3):
        mw.process_request(Request("https://a.example"))
    assert calls == 1

    spider.allowed_domains.append("b.example")
    assert mw.process_request(Request("https://b.example")) is None
    assert calls == 2