Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

35.0 KB
Raw Blame
"""Tests for seed crawls (mwmbl.indexer.seed_crawl): what gets crawled, what counts as new,
and what is recorded for the user. The endpoint that starts a crawl is tested with the
rest of Combined Search, in test_combined_search.py.
"""

import json
from pathlib import Path

import pytest
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.test import Client
from ninja_jwt.tokens import RefreshToken

from mwmbl.indexer import index_batches, seed_crawl, seed_domains
from mwmbl.indexer.index_batches import index_new_documents
from mwmbl.models import SEED_CRAWLER_USERNAME, ApiKey, SeedDomain, SeedDomainCrawl, generate_api_key
from mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndex

User = get_user_model()

SEED = "https://www.tokio.rs/"
# For the rounds run without the worker, which looks the seed crawler's user up itself.
SEED_CRAWLER_ID = 0
STAAN_RESULTS = [
    Document("Tokio", SEED, "An async runtime.", source=DocumentSource.STAAN),
    Document("Async book", "https://rust-lang.github.io/async-book/", "Async Rust.", source=DocumentSource.STAAN),
]


def _page(title, links=()):
    return {"title": title, "extract": f"About {title}.", "links": list(links), "extra_links": []}


# The web the fake crawler sees: url -> content, or None for a fetch that failed.
WEB = {
    SEED: _page(
        "Tokio runtime",
        ["https://tokio.rs/tutorial", "https://elsewhere.example.com/", "https://tokio.rs/search?q=spawn"],
    ),
    "https://tokio.rs/tutorial": _page("Tokio tutorial", ["https://tokio.rs/tutorial/spawning", SEED]),
    "https://tokio.rs/tutorial/spawning": _page("Spawning tasks"),
    "https://elsewhere.example.com/": _page("Elsewhere"),
    "https://tokio.rs/search?q=spawn": _page("Search results"),
    "https://tokio.rs/gone": _page("404 Not Found", ["https://tokio.rs/tutorial"]),
}
# Pages that answer with an error status rather than 200.
STATUSES = {"https://tokio.rs/gone": 404}


@pytest.fixture
def fake_web(monkeypatch, settings):
    """Serve WEB in place of the network, recording each round's batch."""
    settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 0
    rounds = []

    def fake_crawl_batch(urls, num_threads, delay_seconds, redis):
        rounds.append(urls)
        return [
            {"url": url, "status": STATUSES.get(url, 200), "timestamp": 1_700_000_000_000, "content": WEB.get(url)}
            for url in urls
        ]

    monkeypatch.setattr(seed_crawl, "crawl_batch", fake_crawl_batch)
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    monkeypatch.setattr(index_batches, "filter_blacklisted_documents", lambda documents: documents)
    return rounds


@pytest.fixture(autouse=True)
def no_connection_cleanup(monkeypatch):
    """The worker's connection cleanup would close the test's transaction."""
    monkeypatch.setattr(seed_crawl, "close_old_connections", lambda: None)


def _run_rounds(index_path):
    """Run the worker until no crawl is due, returning whether any was.

    The fake web has no delay between a domain's fetches, so a crawl is due again as soon as
    its round is done.
    """
    ran = False
    while seed_crawl.run_next_seed_crawl_round(index_path):
        ran = True
    return ran


@pytest.fixture
def worker():
    return _run_rounds


@pytest.fixture
def index_path(tmp_path):
    path = Path(tmp_path) / "index.tinysearch"
    with TinyIndex.create(Document, str(path), num_pages=64, page_size=4096):
        pass
    return str(path)


@pytest.fixture
def user(db):
    return User.objects.create_user(username="seeder", email="[email protected]", password="x")


def _crawl(seed_urls, domains, is_new=lambda url: True):
    """Run a crawl's rounds to the end, telling it that the pages is_new picks were new."""
    frontier = seed_crawl.Frontier.from_seeds(seed_urls, set(domains))
    crawled = []
    while batch := frontier.next_batch():
        documents = seed_crawl.crawl_round(frontier, batch, redis=None, seed_crawler_id=SEED_CRAWLER_ID)
        crawled.append((len(batch), documents))
        frontier.record_round(batch, {document.url for document in documents if is_new(document.url)})
    return crawled


# ---------------------------------------------------------------------------
# Crawling
# ---------------------------------------------------------------------------


def test_links_are_followed_within_the_domains_and_no_further(fake_web):
    rounds = _crawl([SEED], ["tokio.rs"])

    crawled = [document.url for _, documents in rounds for document in documents]
    assert crawled == [SEED, "https://tokio.rs/tutorial", "https://tokio.rs/tutorial/spawning"]


def test_a_page_is_crawled_once_however_often_it_is_linked(fake_web):
    _crawl([SEED], ["tokio.rs"])

    fetched = [url for batch in fake_web for url in batch]
    assert len(fetched) == len(set(fetched))


def test_each_round_fetches_at_most_one_url_per_domain(fake_web):
    _crawl(
        [SEED, "https://tokio.rs/tutorial", "https://rust-lang.github.io/async-book/"],
        ["tokio.rs", "rust-lang.github.io"],
    )

    assert fake_web[0] == [SEED, "https://rust-lang.github.io/async-book/"]
    assert all(len({url.split("/")[2].removeprefix("www.") for url in batch}) == len(batch) for batch in fake_web)


def test_each_domain_stops_at_its_page_limit(fake_web, settings):
    settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 2

    rounds = _crawl([SEED, "https://rust-lang.github.io/async-book/"], ["tokio.rs", "rust-lang.github.io"])

    fetched = [url for batch in fake_web for url in batch]
    assert fetched == [SEED, "https://rust-lang.github.io/async-book/", "https://tokio.rs/tutorial"]
    assert sum(num_crawled for num_crawled, _ in rounds) == 3


def test_a_domain_is_dropped_once_its_fetches_stop_finding_new_pages(fake_web, monkeypatch, settings):
    settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN = 2
    # An endless site: every page links to the next.
    monkeypatch.setitem(WEB, SEED, _page("Page 0", ["https://tokio.rs/1"]))
    for n in range(1, 20):
        monkeypatch.setitem(WEB, f"https://tokio.rs/{n}", _page(f"Page {n}", [f"https://tokio.rs/{n + 1}"]))

    _crawl([SEED], ["tokio.rs"], is_new=lambda url: False)

    fetched = [url for batch in fake_web for url in batch]
    assert fetched == [SEED, "https://tokio.rs/1"]


def test_a_new_page_resets_a_domains_run_of_stale_fetches(fake_web, monkeypatch, settings):
    settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN = 2
    monkeypatch.setitem(WEB, SEED, _page("Page 0", ["https://tokio.rs/1"]))
    for n in range(1, 20):
        monkeypatch.setitem(WEB, f"https://tokio.rs/{n}", _page(f"Page {n}", [f"https://tokio.rs/{n + 1}"]))

    _crawl([SEED], ["tokio.rs"], is_new=lambda url: url == "https://tokio.rs/1")

    fetched = [url for batch in fake_web for url in batch]
    assert fetched == [SEED, "https://tokio.rs/1", "https://tokio.rs/2", "https://tokio.rs/3"]


def test_a_failed_fetch_adds_no_document(fake_web):
    rounds = _crawl(["https://tokio.rs/missing"], ["tokio.rs"])

    assert rounds == [(1, [])]


def test_an_error_page_is_neither_indexed_nor_followed(fake_web):
    rounds = _crawl(["https://tokio.rs/gone"], ["tokio.rs"])

    assert rounds == [(1, [])]


def test_links_with_a_query_string_are_not_followed(fake_web):
    _crawl([SEED], ["tokio.rs"])

    assert "https://tokio.rs/search?q=spawn" not in [url for batch in fake_web for url in batch]


# ---------------------------------------------------------------------------
# Starting a crawl
# ---------------------------------------------------------------------------


def _queued_jobs(redis_cache):
    """The crawls queued or running, oldest first, with their seeds as yet unfetched."""
    jobs = []
    for record_key in redis_cache.zrange(seed_crawl.READY_KEY, 0, -1):
        record = {key.decode(): value.decode() for key, value in redis_cache.hgetall(record_key).items()}
        frontier = seed_crawl.Frontier.from_json(record["frontier"])
        jobs.append(
            {
                "seed_urls": [url for queue in frontier.queues.values() for url in queue],
                "new_seed_urls": json.loads(record["new_seed_urls"]),
                "domains": json.loads(record["domains"]),
            }
        )
    return jobs


def test_only_staan_results_missing_from_the_index_are_seeds(redis_cache, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    index_pages = [Document("Async book", "https://rust-lang.github.io/async-book/", "")]

    assert (
        seed_crawl.start_seed_crawl(1, "tokio", index_pages, STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED
    )

    [job] = _queued_jobs(redis_cache)
    assert job["seed_urls"] == [SEED]
    # Not retrieved, but Combined Search's write found the index already held it.
    assert job["new_seed_urls"] == []
    # Every Staan domain is crawlable, including those of results the index already had.
    assert job["domains"] == {"rust-lang.github.io": 1, "tokio.rs": 1}


def test_blacklisted_staan_results_are_neither_seeds_nor_domains(redis_cache, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: {SEED})

    seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS, set())

    [job] = _queued_jobs(redis_cache)
    assert SEED not in job["seed_urls"]
    assert job["domains"] == {"rust-lang.github.io": 1}


def test_nothing_to_crawl_starts_nothing(redis_cache, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())

    assert (
        seed_crawl.start_seed_crawl(1, "tokio", STAAN_RESULTS, STAAN_RESULTS, set())
        == seed_crawl.CrawlOutcome.ALREADY_INDEXED
    )
    assert seed_crawl.get_seed_crawl(1, "tokio") is None
    assert _queued_jobs(redis_cache) == []


def test_nothing_starts_when_staan_returned_nothing_crawlable(redis_cache, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: {SEED})

    assert seed_crawl.start_seed_crawl(1, "tokio", [], [STAAN_RESULTS[0]], set()) == seed_crawl.CrawlOutcome.NO_RESULTS
    assert seed_crawl.start_seed_crawl(1, "tokio", [], [], set()) == seed_crawl.CrawlOutcome.NO_RESULTS
    assert _queued_jobs(redis_cache) == []


def test_a_user_has_one_crawl_at_a_time(redis_cache, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())

    assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED
    assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.ALREADY_RUNNING
    assert seed_crawl.start_seed_crawl(1, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.ALREADY_RUNNING
    assert seed_crawl.get_active_seed_crawl_query(1) == "tokio"
    assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED
    assert len(_queued_jobs(redis_cache)) == 2


@pytest.mark.django_db
def test_a_query_already_crawled_is_not_crawled_again(fake_web, redis_cache, user, index_path, worker):
    seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())
    worker(index_path)

    outcome = seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())

    assert outcome == seed_crawl.CrawlOutcome.ALREADY_CRAWLED
    assert _queued_jobs(redis_cache) == []
    assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


def test_nothing_is_queued_once_the_queue_is_full(redis_cache, monkeypatch, settings):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    settings.SEED_CRAWL_MAX_QUEUED = 1

    assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED
    assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.QUEUE_FULL
    assert seed_crawl.get_seed_crawl(2, "tokio") is None


# ---------------------------------------------------------------------------
# Running a crawl
# ---------------------------------------------------------------------------


def _run(user, index_path, seed_urls=(SEED,), new_seed_urls=()):
    """Crawl from the seeds within tokio.rs to the end, as the user's crawl of "tokio"."""
    outcome = seed_crawl.queue_seed_crawl(user.id, "tokio", list(seed_urls), list(new_seed_urls), {"tokio.rs": 1})
    assert outcome == seed_crawl.CrawlOutcome.SCHEDULED
    _run_rounds(index_path)
    return seed_crawl.get_seed_crawl(user.id, "tokio")


def _mark_failed(user, query="tokio"):
    """Mark the user's crawl of the query failed, the one case in which it is crawled again."""
    seed_crawl.get_redis_connection("default").hset(seed_crawl._record_key(user.id, query), "status", "failed")


@pytest.mark.django_db
def test_the_record_lists_the_new_pages_once_the_crawl_is_done(fake_web, redis_cache, user, index_path):
    record = _run(user, index_path)

    assert record["status"] == "done"
    assert record["finished_at"] is not None
    assert record["pages_crawled"] == 3
    assert record["pages_indexed"] == 3
    assert [page["url"] for page in record["pages"]] == [
        SEED,
        "https://tokio.rs/tutorial",
        "https://tokio.rs/tutorial/spawning",
    ]
    assert record["pages"][1] == {
        "url": "https://tokio.rs/tutorial",
        "title": "Tokio tutorial",
        "extract": "About Tokio tutorial.",
    }


@pytest.mark.django_db
def test_the_worker_runs_a_queued_crawl(fake_web, redis_cache, user, index_path, worker):
    seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())

    assert worker(index_path) is True

    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "done"
    # The user can start another crawl once theirs has finished.
    assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


@pytest.mark.django_db
def test_a_failed_crawl_is_recorded_and_not_retried(fake_web, redis_cache, user, index_path, worker, monkeypatch):
    def failing_crawl_batch(urls, num_threads, delay_seconds, redis):
        raise ConnectionError("Redis went away")

    monkeypatch.setattr(seed_crawl, "crawl_batch", failing_crawl_batch)
    seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())

    worker(index_path)

    record = seed_crawl.get_seed_crawl(user.id, "tokio")
    assert record["status"] == "failed"
    assert record["finished_at"] is not None
    assert _queued_jobs(redis_cache) == []
    assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


@pytest.mark.django_db
def test_crawling_a_query_again_keeps_its_pages_and_counts_none_twice(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    _mark_failed(user)

    record = _run(user, index_path)
    user.refresh_from_db()

    assert record["status"] == "done"
    assert record["pages_indexed"] == 3
    assert record["pages_crawled"] == 6
    assert user.seed_search_pages_indexed == 3


@pytest.mark.django_db
def test_the_crawled_pages_are_in_the_index(fake_web, redis_cache, user, index_path):
    _run(user, index_path)

    with TinyIndex(Document, index_path, "r") as index:
        assert "https://tokio.rs/tutorial/spawning" in {document.url for document in index.retrieve("spawning")}


@pytest.mark.django_db
def test_the_crawled_pages_are_credited_to_the_seed_crawler(fake_web, redis_cache, user, index_path):
    """Not to the user who asked for the crawl: the pages were fetched here, not by them."""
    seed_crawler = User.objects.get(username=SEED_CRAWLER_USERNAME)

    _run(user, index_path)

    with TinyIndex(Document, index_path, "r") as index:
        (spawning,) = [d for d in index.retrieve("spawning") if d.url == "https://tokio.rs/tutorial/spawning"]
    assert spawning.user_ids == [seed_crawler.id]


@pytest.mark.django_db
def test_pages_already_in_the_index_are_not_counted_as_new(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    other = User.objects.create_user(username="other", email="[email protected]", password="x")

    record = _run(other, index_path, seed_urls=["https://tokio.rs/tutorial"])

    assert record["pages_crawled"] == 3
    assert record["pages"] == []


def _index_staan_snippet_of_the_seed(index_path):
    """What Combined Search writes before the crawl starts."""
    index_batches.index_documents([Document("Tokio", SEED, "An async runtime.")], index_path)


@pytest.mark.django_db
def test_a_seed_combined_search_found_new_is_counted_once_crawled(fake_web, redis_cache, user, index_path):
    _index_staan_snippet_of_the_seed(index_path)

    record = _run(user, index_path, new_seed_urls=[SEED])

    assert SEED in [page["url"] for page in record["pages"]]


@pytest.mark.django_db
def test_a_seed_the_index_already_held_is_not_counted(fake_web, redis_cache, user, index_path):
    _index_staan_snippet_of_the_seed(index_path)

    record = _run(user, index_path, new_seed_urls=[])

    assert SEED not in [page["url"] for page in record["pages"]]


@pytest.mark.django_db
def test_a_seed_the_blacklist_drops_is_not_counted(fake_web, redis_cache, user, index_path, monkeypatch):
    monkeypatch.setattr(
        index_batches, "filter_blacklisted_documents", lambda documents: [d for d in documents if d.url != SEED]
    )

    record = _run(user, index_path, new_seed_urls=[SEED])

    assert SEED not in [page["url"] for page in record["pages"]]


@pytest.mark.django_db
def test_the_summary_counts_the_pages_without_listing_them(fake_web, redis_cache, user, index_path):
    _run(user, index_path)

    summary = seed_crawl.get_seed_crawl_summary(user.id, "tokio")

    assert summary["pages_indexed"] == 3
    assert "pages" not in summary


@pytest.mark.django_db
def test_new_pages_are_added_to_the_users_total(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    user.refresh_from_db()

    assert user.seed_search_pages_indexed == 3


@pytest.mark.django_db
def test_the_users_total_matches_the_pages_the_crawl_lists(fake_web, redis_cache, user, index_path):
    _index_staan_snippet_of_the_seed(index_path)
    record = _run(user, index_path, new_seed_urls=[SEED])
    user.refresh_from_db()

    assert SEED in [page["url"] for page in record["pages"]]
    assert user.seed_search_pages_indexed == len(record["pages"])


@pytest.mark.django_db
def test_recording_a_seed_search_adds_to_the_users_count(user):
    seed_crawl.record_seed_search(user.id)
    seed_crawl.record_seed_search(user.id)
    user.refresh_from_db()

    assert user.seed_searches == 2


@pytest.mark.django_db
def test_the_users_stats_report_their_seed_search_totals(user):
    User.objects.filter(id=user.id).update(seed_search_pages_indexed=12, seed_searches=4)
    SeedDomain.objects.create(domain="tokio.rs", discovered_by=user)
    SeedDomain.objects.create(domain="smol.rs", discovered_by=user)
    SeedDomain.objects.create(domain="example.com")
    token = str(RefreshToken.for_user(user).access_token)

    response = Client().get("/api/v1/platform/user/stats", HTTP_AUTHORIZATION=f"Bearer {token}")

    assert response.status_code == 200
    assert response.json()["seed_search_pages_indexed"] == 12
    assert response.json()["seed_searches"] == 4
    assert response.json()["seed_sites_discovered"] == 2


# ---------------------------------------------------------------------------
# Progress and seed domains
# ---------------------------------------------------------------------------


def test_a_queued_crawl_has_no_progress_and_no_domains_yet(redis_cache, monkeypatch, db):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS, set())

    summary = seed_crawl.get_seed_crawl_summary(1, "tokio")

    assert summary["progress"] == 0.0
    assert summary["domains"] == []


@pytest.mark.django_db
def test_progress_counts_rounds_against_a_domains_page_limit(redis_cache, user, settings):
    settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 4
    seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())
    record_key = seed_crawl._record_key(user.id, "tokio")
    redis_cache.hset(record_key, mapping={"status": "crawling", "crawl_started_at": seed_crawl._now(), "rounds": 1})

    assert seed_crawl.get_seed_crawl_summary(user.id, "tokio")["progress"] == pytest.approx(0.25, abs=0.01)


@pytest.mark.django_db
def test_a_finished_crawl_reports_its_domains_with_their_scores(fake_web, redis_cache, user, index_path):
    record = _run(user, index_path)

    assert record["progress"] == 1.0
    assert record["domains"] == [
        {
            "domain": "tokio.rs",
            "newly_discovered": True,
            "pages_indexed": 3,
            "new_page_score": 0.15,
            "recent_new_page_score": 0.15,
            "staan_results": 1,
            "score": 0.15,
        }
    ]


@pytest.mark.django_db
def test_the_user_who_first_crawls_a_domain_is_its_discoverer(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    other = User.objects.create_user(username="other", email="[email protected]", password="x")

    record = _run(other, index_path)

    [domain] = record["domains"]
    assert domain["newly_discovered"] is False
    assert domain["staan_results"] == 2
    assert SeedDomain.objects.get(domain="tokio.rs").discovered_by == user


@pytest.mark.django_db
def test_a_domain_the_crawls_have_exhausted_loses_its_score(fake_web, redis_cache, user, index_path, settings):
    settings.SEED_DOMAIN_RECENT_CRAWLS = 2
    _run(user, index_path)
    assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.15

    # The pages are no longer new, so these crawls add none.
    _mark_failed(user)
    _run(user, index_path)
    assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.075
    _mark_failed(user)
    _run(user, index_path)
    seed_domain = SeedDomain.objects.get(domain="tokio.rs")
    assert seed_domain.new_page_score == 0.0
    assert seed_domain.score == 0.0


@pytest.mark.django_db
def test_a_failed_crawl_does_not_score_its_domains(fake_web, redis_cache, user, index_path, monkeypatch):
    def failing_crawl_batch(urls, num_threads, delay_seconds, redis):
        raise ConnectionError("Redis went away")

    monkeypatch.setattr(seed_crawl, "crawl_batch", failing_crawl_batch)

    _run(user, index_path)

    assert SeedDomain.objects.get(domain="tokio.rs").staan_results == 1
    assert not SeedDomainCrawl.objects.exists()


@pytest.mark.django_db
def test_progress_by_time_is_capped_at_one(redis_cache, user):
    seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())
    long_ago = "2020-01-01T00:00:00+00:00"
    redis_cache.hset(
        seed_crawl._record_key(user.id, "tokio"), mapping={"status": "crawling", "crawl_started_at": long_ago}
    )

    assert seed_crawl.get_seed_crawl_summary(user.id, "tokio")["progress"] == 1.0


@pytest.mark.django_db
def test_domains_are_not_reported_until_the_crawl_has_registered_them(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    # A crawl of a fresh query, picked up by the worker but not yet registered.
    seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set())
    redis_cache.hset(
        seed_crawl._record_key(user.id, "rust"), mapping={"status": "crawling", "crawl_started_at": seed_crawl._now()}
    )

    assert seed_crawl.get_seed_crawl_summary(user.id, "rust")["domains"] == []


@pytest.mark.django_db
def test_crawling_a_query_again_reports_only_the_new_crawls_domains(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    _mark_failed(user)

    record = _run(user, index_path)

    [domain] = record["domains"]
    assert domain["newly_discovered"] is False
    assert domain["pages_indexed"] == 0
    assert domain["recent_new_page_score"] == 0.075


@pytest.mark.django_db
def test_a_querys_staan_results_count_once_per_user(fake_web, redis_cache, user, index_path, worker):
    tokio_results = [*STAAN_RESULTS, Document("Tokio docs", "https://tokio.rs/docs", "", source=DocumentSource.STAAN)]

    for _ in range(2):
        seed_crawl.start_seed_crawl(user.id, "tokio", [], tokio_results, set())
        worker(index_path)
        _mark_failed(user)

    assert SeedDomain.objects.get(domain="tokio.rs").staan_results == 2
    assert SeedDomain.objects.get(domain="rust-lang.github.io").staan_results == 1


@pytest.mark.django_db
def test_new_pages_reports_progress_and_domains(fake_web, redis_cache, user, index_path):
    _run(user, index_path)
    token = str(RefreshToken.for_user(user).access_token)

    response = Client().get("/api/v2/combined-search/new-pages/count?q=tokio", HTTP_AUTHORIZATION=f"Bearer {token}")

    assert response.status_code == 200
    assert response.json()["progress"] == 1.0
    assert [domain["domain"] for domain in response.json()["domains"]] == ["tokio.rs"]
    assert "discovered_by" not in response.json()["domains"][0]


SEED_DOMAINS_URL = "/api/v2/combined-search/seed-domains"


def _api_key(user, scope):
    raw, hashed = generate_api_key()
    ApiKey.objects.create(user=user, key=hashed, name="seed-domains", scopes=[scope])
    return raw


def _seed_domains(user, query=""):
    return Client().get(f"{SEED_DOMAINS_URL}{query}", HTTP_X_API_KEY=_api_key(user, ApiKey.Scope.CRAWL))


@pytest.mark.django_db
def test_the_seed_domains_endpoint_lists_the_best_first_without_their_discoverers(redis_cache, user):
    SeedDomain.objects.create(
        domain="low.example.com", discovered_by=user, new_page_score=0.1, staan_results=1, score=0.1
    )
    SeedDomain.objects.create(
        domain="high.example.com", discovered_by=user, new_page_score=0.5, staan_results=4, score=2
    )

    response = _seed_domains(user, "?limit=1")

    assert response.status_code == 200
    assert response.json() == [{"domain": "high.example.com", "new_page_score": 0.5, "staan_results": 4, "score": 2.0}]


@pytest.mark.django_db
def test_the_seed_domains_endpoint_requires_a_crawl_scoped_key(redis_cache, user):
    search_key = _api_key(user, ApiKey.Scope.SEARCH)

    assert Client().get(SEED_DOMAINS_URL).status_code == 401
    assert Client().get(SEED_DOMAINS_URL, HTTP_X_API_KEY=search_key).status_code == 401


@pytest.mark.django_db
def test_seed_domains_are_published_in_batches(redis_cache, user):
    SeedDomain.objects.create(domain="old.example.com", new_page_score=0.1, staan_results=1, score=0.1)
    assert [domain["domain"] for domain in _seed_domains(user).json()] == ["old.example.com"]

    # What a crawl changes is not shown until the list is next published.
    SeedDomain.objects.create(domain="new.example.com", new_page_score=0.5, staan_results=1, score=0.5)
    SeedDomain.objects.filter(domain="old.example.com").update(score=0.0)
    assert _seed_domains(user).json() == [
        {"domain": "old.example.com", "new_page_score": 0.1, "staan_results": 1, "score": 0.1}
    ]

    cache.delete(seed_domains.PUBLISHED_KEY)
    assert [domain["domain"] for domain in _seed_domains(user).json()] == ["new.example.com", "old.example.com"]


def test_index_new_documents_returns_only_urls_the_index_lacked(index_path, monkeypatch):
    monkeypatch.setattr(index_batches, "filter_blacklisted_documents", lambda documents: documents)
    first = Document("Tokio tutorial", "https://tokio.rs/tutorial", "Learn tokio.")
    second = Document("Spawning tasks", "https://tokio.rs/spawning", "Spawning with tokio.")

    assert index_new_documents([first], index_path).new == {first.url}
    indexed = index_new_documents([first, second], index_path)
    assert indexed.stored == {first.url, second.url}
    assert indexed.new == {second.url}


def test_index_new_documents_counts_nothing_the_blacklist_drops(index_path, monkeypatch):
    monkeypatch.setattr(index_batches, "filter_blacklisted_documents", lambda documents: [])

    indexed = index_new_documents([Document("Tokio", SEED, "Tokio.")], index_path)

    assert indexed == (set(), set())


# ---------------------------------------------------------------------------
# Workers that die, stall or take turns
# ---------------------------------------------------------------------------


def _claim(redis_cache):
    """Claim the crawl due soonest as a worker would, but run none of it."""
    return redis_cache.eval(
        seed_crawl._CLAIM_SCRIPT,
        1,
        seed_crawl.READY_KEY,
        seed_crawl.time.time(),
        seed_crawl.settings.SEED_CRAWL_LEASE_SECONDS,
        seed_crawl.settings.SEED_CRAWL_MAX_ATTEMPTS,
        seed_crawl._now(),
    )


@pytest.mark.django_db
def test_a_crawl_whose_worker_died_is_resumed_once_its_lease_runs_out(
    fake_web, redis_cache, user, index_path, worker, settings
):
    settings.SEED_CRAWL_LEASE_SECONDS = 0
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    # A worker claims the crawl and dies before writing anything.
    _claim(redis_cache)

    worker(index_path)

    record = seed_crawl.get_seed_crawl(user.id, "tokio")
    assert record["status"] == "done"
    assert record["pages_indexed"] == 3


@pytest.mark.django_db
def test_a_crawl_is_not_claimed_again_while_its_lease_lasts(fake_web, redis_cache, user, index_path, worker):
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    _claim(redis_cache)

    assert worker(index_path) is False
    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"


@pytest.mark.django_db
def test_a_worker_that_lost_its_claim_writes_nothing(redis_cache, user, settings):
    settings.SEED_CRAWL_LEASE_SECONDS = 0
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    record_key, stale_token = _claim(redis_cache)
    # Too slow: another worker has claimed the crawl since.
    _claim(redis_cache)

    seed_crawl._finish(redis_cache, record_key.decode(), stale_token, user.id, "failed")

    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"
    assert seed_crawl.get_active_seed_crawl_query(user.id) == "tokio"


@pytest.mark.django_db
def test_a_round_that_keeps_killing_its_workers_fails_the_crawl(redis_cache, user, settings, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    settings.SEED_CRAWL_LEASE_SECONDS = 0
    settings.SEED_CRAWL_MAX_ATTEMPTS = 2
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})

    assert _claim(redis_cache) is not None
    assert _claim(redis_cache) is not None
    assert _claim(redis_cache) is None

    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"
    assert redis_cache.zcard(seed_crawl.READY_KEY) == 0
    assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


@pytest.mark.django_db
def test_queued_crawls_run_one_after_another_oldest_first(fake_web, redis_cache, user, index_path, worker):
    other = User.objects.create_user(username="other", email="[email protected]", password="x")
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    seed_crawl.queue_seed_crawl(other.id, "tokio", ["https://tokio.rs/tutorial/spawning"], [], {"tokio.rs": 1})

    worker(index_path)

    # The first crawl to the end, then the second.
    assert fake_web == [
        [SEED],
        ["https://tokio.rs/tutorial"],
        ["https://tokio.rs/tutorial/spawning"],
        ["https://tokio.rs/tutorial/spawning"],
    ]
    assert seed_crawl.get_seed_crawl(other.id, "tokio")["status"] == "done"


@pytest.mark.django_db
def test_a_crawls_rounds_are_spaced_by_the_domain_delay(fake_web, redis_cache, user, index_path, settings, monkeypatch):
    settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 60
    sleeps = []
    monkeypatch.setattr(seed_crawl.time, "sleep", sleeps.append)
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})

    seed_crawl.run_next_seed_crawl_round(index_path)
    seed_crawl.run_next_seed_crawl_round(index_path)

    assert sleeps == [pytest.approx(60, abs=1)]


@pytest.mark.django_db
def test_a_crawl_left_running_before_crawls_were_leased_is_reported_failed_and_can_be_redone(
    redis_cache, user, monkeypatch
):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    redis_cache.hset(
        seed_crawl._record_key(user.id, "tokio"),
        mapping={"status": "crawling", "query": "tokio", "started_at": seed_crawl._now(), "pages_crawled": 4},
    )

    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"
    assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


@pytest.mark.django_db
def test_a_crawl_whose_record_has_expired_is_dropped_and_the_next_one_claimed(
    fake_web, redis_cache, user, index_path, worker
):
    other = User.objects.create_user(username="other", email="[email protected]", password="x")
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    seed_crawl.queue_seed_crawl(other.id, "tokio", [SEED], [], {"tokio.rs": 1})
    redis_cache.delete(seed_crawl._record_key(user.id, "tokio"))

    worker(index_path)

    assert redis_cache.zcard(seed_crawl.READY_KEY) == 0
    assert not redis_cache.exists(seed_crawl._record_key(user.id, "tokio"))
    assert seed_crawl.get_seed_crawl(other.id, "tokio")["status"] == "done"


@pytest.mark.django_db
def test_an_active_crawl_whose_record_has_expired_blocks_nothing(redis_cache, user, monkeypatch):
    monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    redis_cache.delete(seed_crawl._record_key(user.id, "tokio"))
    redis_cache.delete(seed_crawl.READY_KEY)

    assert seed_crawl.get_active_seed_crawl_query(user.id) is None
    assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED


@pytest.mark.django_db
def test_the_users_active_crawl_expires_with_its_record(redis_cache, user, settings):
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})

    ttl = redis_cache.ttl(seed_crawl._active_key(user.id))

    assert 0 < ttl <= settings.SEED_CRAWL_RECORD_TTL_SECONDS


@pytest.mark.django_db
def test_a_crawl_out_of_attempts_gives_way_to_the_next_due(redis_cache, user, settings):
    other = User.objects.create_user(username="other", email="[email protected]", password="x")
    settings.SEED_CRAWL_LEASE_SECONDS = 0
    settings.SEED_CRAWL_MAX_ATTEMPTS = 1
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
    _claim(redis_cache)
    seed_crawl.queue_seed_crawl(other.id, "tokio", [SEED], [], {"tokio.rs": 1})

    record_key, _ = _claim(redis_cache)

    assert record_key.decode() == seed_crawl._record_key(other.id, "tokio")
    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"
    assert seed_crawl.get_active_seed_crawl_query(user.id) is None


@pytest.mark.django_db
def test_a_done_crawl_stays_done_if_scoring_its_domains_fails(fake_web, redis_cache, user, index_path, monkeypatch):
    def fail(pages_indexed):
        raise RuntimeError("database gone")

    monkeypatch.setattr(seed_crawl, "record_seed_domain_crawls", fail)
    seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})

    _run_rounds(index_path)

    assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "done"