Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

5.9 KB
Raw Blame
from __future__ import annotations

import asyncio

import httpx
import pytest

from deeptutor.services.web_source import crawler, robots


@pytest.fixture
def crawl_clock(monkeypatch):
    now = [100.0]
    monkeypatch.setattr(robots.time, "monotonic", lambda: now[0])

    async def sleep(delay):
        now[0] += delay

    monkeypatch.setattr(robots.asyncio, "sleep", sleep)
    monkeypatch.setattr(robots, "_is_disallowed_host", lambda _: False)
    monkeypatch.setattr(crawler, "_is_disallowed_host", lambda _: False)
    return now


def test_specific_agent_groups_merge_and_support_wildcards_queries_and_escaped_paths():
    policy = robots.parse_robots_txt("""
User-agent: *
Disallow: /
User-agent: DeepTutor
Disallow: /docs/*?private=*
Disallow: /caf%C3%A9
Allow: /docs/open$
Crawl-delay: nan
User-agent: OtherBot
Disallow: /
User-agent: deeptutor
Disallow: /docs/open
Allow: /docs/open
Disallow: /%7Eprivate
Crawl-delay: 2
""")
    assert policy.crawl_delay_s == 2
    assert policy.permits("https://example.com/docs/open")
    assert policy.permits("https://example.com/docs/public")
    assert not policy.permits("https://example.com/docs/x?private=yes")
    assert not policy.permits("https://example.com/café")
    assert not policy.permits("https://example.com/~private")


@pytest.mark.asyncio
async def test_crawl_skips_disallowed_links_and_paces_concurrent_siblings(crawl_clock):
    requested = []

    def handler(request):
        requested.append((request.url.path, crawl_clock[0]))
        if request.url.path == "/robots.txt":
            return httpx.Response(
                200, text="User-agent: *\nDisallow: /docs/private\nCrawl-delay: 2"
            )
        return httpx.Response(
            200,
            text='<html><h1>Guide</h1><a href="/docs/private">private</a><a href="/docs/a">a</a><a href="/docs/b">b</a></html>',
        )

    result = await crawler.crawl_docs_site(
        "https://example.com/docs/",
        max_depth=1,
        client_factory=lambda: httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    assert len(result.pages) == 3
    assert [path for path, _ in requested] == ["/robots.txt", "/docs/", "/docs/a", "/docs/b"]
    assert all(b[1] - a[1] >= 2 for a, b in zip(requested, requested[1:]))


@pytest.mark.asyncio
@pytest.mark.parametrize("status,expected", [(404, True), (503, False), (429, False)])
async def test_missing_robots_allows_pages_but_unreachable_policy_blocks(
    crawl_clock, status, expected
):
    requested = []

    def handler(request):
        requested.append(request.url.path)
        return httpx.Response(
            status if request.url.path == "/robots.txt" else 200, text="<h1>Docs</h1>"
        )

    result = await crawler.crawl_docs_site(
        "https://example.com/docs/",
        max_depth=0,
        client_factory=lambda: httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    assert bool(result.pages) is expected
    assert ("/docs/" in requested) is expected
    if not expected:
        assert result.errors == ["robots.txt unavailable; crawl blocked"]


@pytest.mark.asyncio
async def test_redirect_target_uses_its_own_origin_policy(crawl_clock):
    requested = []

    def handler(request):
        requested.append(str(request.url))
        if request.url.path == "/robots.txt":
            rule = "Disallow: /private" if request.url.host == "other.example.com" else ""
            return httpx.Response(200, text="User-agent: *\n" + rule)
        return httpx.Response(302, headers={"Location": "https://other.example.com/private"})

    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
        assert (
            await crawler._fetch_page(
                "https://example.com/docs", client=client, access=robots.CrawlAccess(client)
            )
            is None
        )
    assert requested == [
        "https://example.com/robots.txt",
        "https://example.com/docs",
        "https://other.example.com/robots.txt",
    ]


@pytest.mark.asyncio
async def test_robots_redirect_cannot_contact_private_network(crawl_clock, monkeypatch):
    requested = []
    monkeypatch.setattr(robots, "_is_disallowed_host", lambda host: host == "127.0.0.1")

    def handler(request):
        requested.append(str(request.url))
        return httpx.Response(302, headers={"Location": "http://127.0.0.1/robots.txt"})

    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
        policy = await robots.CrawlAccess(client).policy_for("https://example.com/docs")
    assert not policy.available
    assert requested == ["https://example.com/robots.txt"]


@pytest.mark.asyncio
async def test_concurrent_requests_fetch_robots_once_and_keep_minimum_pacing(crawl_clock):
    requested = []

    async def handler(request):
        requested.append((request.url.path, crawl_clock[0]))
        return httpx.Response(
            404 if request.url.path == "/robots.txt" else 200, text="<h1>Docs</h1>"
        )

    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
        access = robots.CrawlAccess(client)
        await asyncio.gather(
            *(
                crawler._fetch_page(f"https://example.com/docs/{i}", client=client, access=access)
                for i in range(4)
            )
        )
    assert [path for path, _ in requested].count("/robots.txt") == 1
    assert all(
        b[1] - a[1] >= robots.DEFAULT_REQUEST_INTERVAL_S for a, b in zip(requested, requested[1:])
    )


def test_absurd_crawl_delay_is_ignored_not_honored() -> None:
    """A hostile robots.txt can otherwise park every request to this host on
    one unbounded sleep; RFC 9309 lets crawlers cap unreasonable values."""
    policy = robots.parse_robots_txt(
        """
User-agent: *
Disallow: /private
Crawl-delay: 1000000
User-agent: deeptutor
Crawl-delay: 45
"""
    )
    assert policy.crawl_delay_s == 45
    assert policy.permits("https://example.com/docs/open")