Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

5.0 KB
Raw Blame
"""
Recording the URLs found in crawled batches: which links are kept, who gets credit
for them, and what is queued.
"""

from collections import defaultdict
from datetime import datetime, timezone
from unittest.mock import MagicMock

import pytest

from mwmbl.crawler.batch import HashedBatch, Item, ItemContent, ItemError, Link
from mwmbl.crawler.urls import URLStatus
from mwmbl.indexer import update_urls
from mwmbl.indexer.update_urls import get_datetime_from_timestamp, process_link, record_urls_in_database

TIMESTAMP = datetime(2024, 1, 1, tzinfo=timezone.utc)


class FakeBlacklist:
    def is_domain_blacklisted(self, domain: str) -> bool:
        return domain == "blocked.example"


def _process(link_url: str):
    url_users, url_timestamps = {}, {}
    domain_links = defaultdict(set)
    process_link(
        "user", "source.example", Link(url=link_url, link_type="content"), TIMESTAMP,
        url_timestamps, url_users, FakeBlacklist(), domain_links,
    )  # fmt: skip
    return url_users, url_timestamps, domain_links


def test_timestamp_is_converted_to_an_aware_datetime():
    assert get_datetime_from_timestamp(1704067200.0) == TIMESTAMP


def test_a_link_credits_the_user_and_its_root_url():
    url_users, url_timestamps, domain_links = _process("https://target.example/page")

    assert url_users == {"https://target.example/page": "user", "https://target.example/": "user"}
    assert set(url_timestamps) == set(url_users)
    assert domain_links == {"source.example": {"target.example"}}


@pytest.mark.parametrize("link_url", ["https://blocked.example/page", "not a url"])
def test_unparseable_and_blacklisted_links_are_dropped(link_url):
    url_users, url_timestamps, domain_links = _process(link_url)

    assert url_users == {}
    assert url_timestamps == {}
    assert domain_links == {}


@pytest.fixture
def recorded(monkeypatch):
    """Patch the database and blacklist, returning the URLs that would have been queued."""
    found_urls = []
    domain_link_db = MagicMock()
    domain_link_db.__enter__.return_value = domain_link_db

    monkeypatch.setattr(update_urls, "get_default_blacklist_provider", lambda curated: FakeBlacklist())
    monkeypatch.setattr(update_urls, "_add_found_urls_to_db_and_queue", lambda urls, queue: found_urls.extend(urls))
    monkeypatch.setattr(update_urls, "DomainLinkDatabase", lambda: domain_link_db)
    return found_urls, domain_link_db


def _item(url, content=None, status=200, error=None):
    return Item(url=url, status=status, timestamp=1704067200000.0, content=content, error=error)


def test_recording_a_batch_classifies_every_url(recorded):
    found_urls, domain_link_db = recorded
    content = ItemContent(title="T", extract="E", links=["https://linked.example/a"])
    batch = HashedBatch(
        user_id_hash="hash",
        timestamp=1704067200.0,
        items=[
            _item("https://ok.example/", content=content),
            _item("https://gone.example/", status=404),
            _item("https://slow.example/", error=ItemError(name="AbortError")),
            _item("https://robots.example/", error=ItemError(name="RobotsDenied")),
            _item("https://odd.example/", status=500),
        ],
    )

    record_urls_in_database([batch], MagicMock())

    statuses = {found.url: found.status for found in found_urls}
    assert statuses == {
        "https://ok.example/": URLStatus.CRAWLED,
        "https://gone.example/": URLStatus.ERROR_404,
        "https://slow.example/": URLStatus.ERROR_TIMEOUT,
        "https://robots.example/": URLStatus.ERROR_ROBOTS_DENIED,
        "https://odd.example/": URLStatus.ERROR_OTHER,
        "https://linked.example/a": URLStatus.NEW,
        "https://linked.example/": URLStatus.NEW,
    }
    assert {found.user_id_hash for found in found_urls} == {"hash"}
    domain_link_db.update_domain_links.assert_called_once_with("ok.example", {"linked.example"})


def test_a_crawled_page_with_an_unparseable_url_contributes_no_links(recorded):
    found_urls, domain_link_db = recorded
    content = ItemContent(title="T", extract="E", links=["https://linked.example/a"])
    batch = HashedBatch(user_id_hash="hash", timestamp=1704067200.0, items=[_item("no-scheme", content=content)])

    record_urls_in_database([batch], MagicMock())

    assert [found.url for found in found_urls] == ["no-scheme"]
    domain_link_db.update_domain_links.assert_not_called()


def test_hn_links_are_added_until_the_domain_limit(monkeypatch):
    queue = MagicMock()
    queue.get_domain_count.side_effect = [0, 100, 200]
    monkeypatch.setattr(update_urls, "get_domain_max_urls", lambda domain, curated: 150)
    monkeypatch.setattr(update_urls.requests, "get", lambda url: MagicMock(json=lambda: 1000))
    added = []
    monkeypatch.setattr(update_urls, "_add_found_urls_to_db_and_queue", lambda urls, q: added.append(urls))

    update_urls.add_hn_links(queue)

    assert len(added) == 2
    assert added[0][0].url == "https://news.ycombinator.com/item?id=1000"
    assert added[1][0].url == "https://news.ycombinator.com/item?id=900"