Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

5.2 KB
Raw Blame
"""
Regression tests for URL de-duplication.

A freshly-crawled URL (or one that failed, e.g. robots.txt denied) must be
recorded as crawled *now* so that it is not immediately re-queued and crawled
again. The bug these tests guard against was that `update_found_urls` derived
`last_crawled` only from the bloom-filter state *before* the current batch, so a
first-time crawl came back with `last_crawled=None` and `queue_urls` treated it
as never-crawled.
"""

import glob
import os
from datetime import datetime, timezone

import fakeredis
import pytest

from mwmbl.crawler.urls import FoundURL, URLDatabase, URLStatus
from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
from mwmbl.indexer.update_urls import get_datetime_from_timestamp
from mwmbl.redis_url_queue import RedisURLQueue


@pytest.fixture
def clean_bloom_filters():
    """Remove any bloom filter files so each test starts from a clean slate."""

    def _remove():
        for path in glob.glob("/tmp/test_urls*.bloom"):
            os.remove(path)

    _remove()
    yield
    _remove()


def test_crawled_url_marked_as_recently_crawled(clean_bloom_filters):
    """A first-time crawled URL should come back with last_crawled set to the crawl time."""
    crawled_at = datetime.now(timezone.utc)
    found = FoundURL(
        url="https://example.com/page",
        user_id_hash="user",
        status=URLStatus.CRAWLED,
        timestamp=crawled_at,
        last_crawled=None,
    )
    with URLDatabase() as url_db:
        new_urls = url_db.update_found_urls([found])

    assert len(new_urls) == 1
    assert new_urls[0].last_crawled == crawled_at


def test_robots_denied_url_marked_as_recently_crawled(clean_bloom_filters):
    """A robots-denied URL must also be marked crawled so it isn't re-attempted."""
    crawled_at = datetime.now(timezone.utc)
    found = FoundURL(
        url="https://example.com/blocked",
        user_id_hash="user",
        status=URLStatus.ERROR_ROBOTS_DENIED,
        timestamp=crawled_at,
        last_crawled=None,
    )
    with URLDatabase() as url_db:
        new_urls = url_db.update_found_urls([found])

    assert new_urls[0].last_crawled == crawled_at


def test_crawled_url_is_not_requeued(clean_bloom_filters):
    """End-to-end: a just-crawled URL is not put back on the queue, but a newly
    discovered link is."""
    crawled = FoundURL(
        url="https://crawled.example/page",
        user_id_hash="user",
        status=URLStatus.CRAWLED,
        timestamp=datetime.now(timezone.utc),
        last_crawled=None,
    )
    new_link = FoundURL(
        url="https://discovered.example/page",
        user_id_hash="user",
        status=URLStatus.NEW,
        timestamp=datetime.now(timezone.utc),
        last_crawled=None,
    )

    with URLDatabase() as url_db:
        new_urls = url_db.update_found_urls([crawled, new_link])

    redis = fakeredis.FakeRedis(decode_responses=True)
    url_queue = RedisURLQueue(redis, lambda: set(), StaticBlacklistProvider(set()))
    url_queue.queue_urls(new_urls)

    assert url_queue.get_domain_count("crawled.example") == 0
    assert url_queue.get_domain_count("discovered.example") == 1


def test_crawl_date_is_the_crawl_time_not_the_start_of_the_month(clean_bloom_filters):
    """The bloom filters only record the month a URL was crawled in. Reporting the start
    of that month as the crawl date makes a URL crawled on the 31st look 30 days old -
    exactly the age at which queue_urls hands it out again."""
    crawled_at = datetime(2026, 8, 31, 12, 0, 0, tzinfo=timezone.utc)
    found = FoundURL(
        url="https://end-of-month.example/page",
        user_id_hash="user",
        status=URLStatus.CRAWLED,
        timestamp=crawled_at,
        last_crawled=None,
    )
    with URLDatabase() as url_db:
        new_urls = url_db.update_found_urls([found])

    assert new_urls[0].last_crawled == crawled_at


def test_crawl_date_from_a_real_batch_timestamp_is_aware(clean_bloom_filters):
    """Batch timestamps are timezone-aware, and so is the clock queue_urls subtracts
    last_crawled from: mixing an aware and a naive datetime raises."""
    crawled_at = get_datetime_from_timestamp(datetime.now(timezone.utc).timestamp())
    found = FoundURL(
        url="https://aware.example/page",
        user_id_hash="user",
        status=URLStatus.CRAWLED,
        timestamp=crawled_at,
        last_crawled=None,
    )
    with URLDatabase() as url_db:
        new_urls = url_db.update_found_urls([found])

    assert new_urls[0].last_crawled.tzinfo == timezone.utc
    assert new_urls[0].last_crawled == crawled_at

    redis = fakeredis.FakeRedis(decode_responses=True)
    url_queue = RedisURLQueue(redis, lambda: set(), StaticBlacklistProvider(set()))
    url_queue.queue_urls(new_urls)

    assert url_queue.get_domain_count("aware.example") == 0


def test_url_queue_gives_its_curated_domains_to_the_default_blacklist():
    """The standalone crawler has no database and fetches approved domains over HTTP, so
    the queue's own source of curated domains is what has to reach the blacklist."""
    redis = fakeredis.FakeStrictRedis(decode_responses=True)
    url_queue = RedisURLQueue(redis, lambda: {"pudding.cool"})

    assert url_queue.blacklist_provider.get_exempt_domains() == {"pudding.cool"}