Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

14.1 KB
Raw Blame
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch

import fakeredis
import pytest

from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
from mwmbl.indexer.blacklist_snapshot import SnapshotBlacklist
from mwmbl.indexer.index_batches import (
    _merge_user_ids,
    combine_documents,
    index_documents,
    index_results_against_query,
    sort_documents,
)
from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentState, TinyIndex


class UrlRanker:
    @staticmethod
    def order_results(terms: list[str], pages: list[Document], is_complete: bool):
        return sorted(pages, key=lambda doc: doc.url)


def test_sort_documents():
    existing_documents = [
        Document(title="title1", url="1", extract="extract1", term="term1"),
        Document(title="title2", url="4", extract="extract2", term="term2"),
        Document(title="title3", url="6", extract="extract3", term="term3"),
        Document(title="title4", url="5", extract="extract4", term="term3"),
    ]

    documents = [
        Document(title="title5", url="2", extract="extract5", term="term1"),
        Document(title="title6", url="3", extract="extract6", term="term2"),
    ]

    # Sort the documents
    sorted_documents = sort_documents(documents, existing_documents, UrlRanker())

    # Existing terms without new documents should not be sorted
    assert sorted_documents == [
        Document(title="title1", url="1", extract="extract1", term="term1"),
        Document(title="title6", url="3", extract="extract6", term="term2"),
        Document(title="title3", url="6", extract="extract3", term="term3"),
        Document(title="title5", url="2", extract="extract5", term="term1"),
        Document(title="title2", url="4", extract="extract2", term="term2"),
        Document(title="title4", url="5", extract="extract4", term="term3"),
    ]


def test_sort_documents_curated_items_first():
    existing_documents = [
        Document(title="title1", url="1", extract="extract1", term="term1"),
        Document(title="title2", url="4", extract="extract2", term="term2"),
        Document(title="title3", url="6", extract="extract3", term="term1", state=DocumentState.ORGANIC_APPROVED),
        Document(title="title4", url="5", extract="extract4", term="term2", state=DocumentState.ORGANIC_APPROVED),
    ]

    documents = [
        Document(title="title5", url="2", extract="extract5", term="term1"),
        Document(title="title6", url="3", extract="extract6", term="term2"),
    ]

    # Sort the documents
    sorted_documents = sort_documents(documents, existing_documents, UrlRanker())

    # Curated items should be first
    assert sorted_documents == [
        Document(title="title3", url="6", extract="extract3", term="term1", state=DocumentState.ORGANIC_APPROVED),
        Document(title="title4", url="5", extract="extract4", term="term2", state=DocumentState.ORGANIC_APPROVED),
        Document(title="title1", url="1", extract="extract1", term="term1"),
        Document(title="title6", url="3", extract="extract6", term="term2"),
        Document(title="title5", url="2", extract="extract5", term="term1"),
        Document(title="title2", url="4", extract="extract2", term="term2"),
    ]


def test_sort_documents_duplicates_keep_synced_state():
    existing_documents = [
        Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),
    ]

    documents = [
        Document(title="title1", url="1", extract="extract1", term="term1"),
    ]

    # Sort the documents
    combined_documents = combine_documents(documents, existing_documents, mark_synced=False, ranker=UrlRanker())
    print("Combined documents", combined_documents)

    # Curated items should be first
    assert combined_documents == [
        Document(title="title1", url="1", extract="extract1", term="term1", state=DocumentState.SYNCED_WITH_MAIN_INDEX),
    ]


# ---------------------------------------------------------------------------
# index_results_against_query
# ---------------------------------------------------------------------------


def test_index_results_against_query():
    # "rust", "async" and the bigram "rust async" land on distinct pages here,
    # so cross-term URL dedup within a page does not interfere with the asserts.
    num_pages = 64
    a = Document(title="Rust async runtime", url="http://a.example/page", extract="an async runtime")
    b = Document(title="Rust systems guide", url="http://b.example", extract="low level")
    c = Document(title="Async patterns", url="http://c.example", extract="concurrency primitives")
    docs = [a, b, c]

    with TemporaryDirectory() as temp_dir:
        index_path = str(Path(temp_dir) / "temp-index.tinysearch")
        with TinyIndex.create(Document, index_path, num_pages=num_pages, page_size=4096):
            pass

        new_count = index_results_against_query(docs, "rust async", index_path)

        # All three pages are newly added (each matches at least one term).
        assert new_count == 3

        with TinyIndex(Document, index_path, "r") as indexer:
            rust_urls = {d.url for d in indexer.retrieve("rust")}
            async_urls = {d.url for d in indexer.retrieve("async")}
            bigram_urls = {d.url for d in indexer.retrieve("rust async")}

        # Unigram "rust" matches A and B; "async" matches A and C.
        assert rust_urls == {a.url, b.url}
        assert async_urls == {a.url, c.url}
        # The bigram needs both words present, so only A matches.
        assert bigram_urls == {a.url}

        # Re-indexing the same results adds nothing new.
        assert index_results_against_query(docs, "rust async", index_path) == 0


def test_index_results_against_query_keeps_title_only_documents():
    # Title-only results (empty extract) must still be indexed, matching what
    # Super Search now keeps for display. The term is matched via the URL/title
    # token set, so an empty extract should not exclude the document.
    num_pages = 64
    doc = Document(title="Kitsas dictionary", url="https://en.wiktionary.org/wiki/kitsas", extract="")
    docs = [doc]

    with TemporaryDirectory() as temp_dir:
        index_path = str(Path(temp_dir) / "temp-index.tinysearch")
        with TinyIndex.create(Document, index_path, num_pages=num_pages, page_size=4096):
            pass

        new_count = index_results_against_query(docs, "kitsas", index_path)
        assert new_count == 1

        with TinyIndex(Document, index_path, "r") as indexer:
            kitsas_urls = {d.url for d in indexer.retrieve("kitsas")}
        assert kitsas_urls == {doc.url}


# ---------------------------------------------------------------------------
# _merge_user_ids
# ---------------------------------------------------------------------------


def test_merge_user_ids_empty_existing():
    assert _merge_user_ids(None, [1]) == [1]


def test_merge_user_ids_basic():
    assert _merge_user_ids([1], [2]) == [1, 2]


def test_merge_user_ids_deduplication_moves_to_end():
    # User already present moves to most-recent position
    assert _merge_user_ids([1, 2], [1]) == [2, 1]


def test_merge_user_ids_capped_at_max():
    assert _merge_user_ids([1, 2], [3]) == [2, 3]


def test_merge_user_ids_both_none():
    assert _merge_user_ids(None, None) is None


# ---------------------------------------------------------------------------
# combine_documents: user_ids and last_crawled merging
# ---------------------------------------------------------------------------


def test_combine_documents_merges_user_ids_for_same_url():
    existing = [Document(title="t", url="http://a.com", extract="e", term="q", user_ids=[1])]
    new_docs = [Document(title="t", url="http://a.com", extract="e", term="q", user_ids=[2])]
    combined = combine_documents(existing, new_docs, mark_synced=False, ranker=UrlRanker())
    assert len(combined) == 1
    assert set(combined[0].user_ids) == {1, 2}


def test_combine_documents_uses_max_last_crawled():
    existing = [Document(title="t", url="http://a.com", extract="e", term="q", last_crawled=1000)]
    new_docs = [Document(title="t", url="http://a.com", extract="e", term="q", last_crawled=2000)]
    combined = combine_documents(existing, new_docs, mark_synced=False, ranker=UrlRanker())
    assert combined[0].last_crawled == 2000


def test_combine_documents_propagates_user_ids_to_winner():
    """When multiple docs share a URL, whichever wins carries the merged user_ids."""
    existing = [Document(title="old", url="http://a.com", extract="e1", term="q", user_ids=[1])]
    new_docs = [Document(title="new", url="http://a.com", extract="e2", term="q", user_ids=[2])]
    combined = combine_documents(existing, new_docs, mark_synced=False, ranker=UrlRanker())
    assert len(combined) == 1
    assert 1 in combined[0].user_ids
    assert 2 in combined[0].user_ids


@pytest.mark.parametrize(
    "existing_ids, existing_extract, incoming_extract, expected",
    [
        ([1], "a longer extract", "short", [3, 1]),
        ([1], "short", "a longer extract", [1, 3]),
        # The kept copy's ids keep their order, its author last, when the cap drops the rest.
        ([1, 2], "a longer extract", "short", [1, 2]),
    ],
)
def test_combine_documents_puts_the_kept_texts_author_last(existing_ids, existing_extract, incoming_extract, expected):
    """mwmbl.indexer.agreement reads the author of a stored copy's text off the end of its
    user_ids, whichever copy ranks first."""
    existing = [Document(title="t", url="http://a.com", extract=existing_extract, term="q", user_ids=existing_ids)]
    new_docs = [Document(title="t", url="http://a.com", extract=incoming_extract, term="q", user_ids=[3])]

    (kept,) = combine_documents(existing, new_docs, mark_synced=False, ranker=LongestExtractRanker())

    assert kept.user_ids == expected


class LongestExtractRanker:
    @staticmethod
    def order_results(terms: list[str], pages: list[Document], is_complete: bool):
        return sorted(pages, key=lambda doc: -len(doc.extract))


# ---------------------------------------------------------------------------
# index_documents: blacklist enforcement
# ---------------------------------------------------------------------------
#
# index_documents() is the common choke point for every path that adds content to the
# index (offline batch processing, the trusted-crawler POST /results endpoint, the
# standalone crawl tool) - crawling/link-discovery only stop *new* crawling of a
# blacklisted domain, they don't stop a submitted batch that already contains one of its
# pages, so indexing needs its own blacklist check.

PATCH_TARGET = "mwmbl.indexer.index_batches.get_snapshot_blacklist"


def snapshot_blacklist(domains: set[str]) -> SnapshotBlacklist:
    """A blacklist backed by the given domains alone.

    index_documents() reads the published snapshot rather than constructing the remote
    providers, so the double it gets has to be a SnapshotBlacklist. Supplying the domains
    as its built-in rules keeps these tests off the snapshot machinery, which
    test_blacklist_snapshot.py covers.
    """
    return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(domains), redis_client=fakeredis.FakeRedis())


@pytest.fixture
def index_path(tmp_path):
    path = tmp_path / "test.tinysearch"
    TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=10, page_size=PAGE_SIZE)
    return str(path)


def _all_urls(index_path, num_pages=10):
    urls = []
    with TinyIndex(item_factory=Document, index_path=index_path, mode="r") as index:
        for page_index in range(num_pages):
            urls.extend(doc.url for doc in index.get_page(page_index))
    return urls


def test_index_documents_skips_blacklisted_domain(index_path):
    documents = [
        Document(title="Bad", url="https://fineartteens.com/x", extract="teen gallery"),
        Document(title="Good", url="https://example.com/y", extract="a good page"),
    ]

    with patch(PATCH_TARGET, return_value=snapshot_blacklist({"fineartteens.com"})):
        index_documents(documents, index_path)

    urls = _all_urls(index_path)
    assert "https://fineartteens.com/x" not in urls
    assert "https://example.com/y" in urls


def test_index_documents_keeps_everything_when_nothing_blacklisted(index_path):
    documents = [Document(title="Good", url="https://example.com/y", extract="a good page")]

    with patch(PATCH_TARGET, return_value=snapshot_blacklist(set())):
        index_documents(documents, index_path)

    assert "https://example.com/y" in _all_urls(index_path)


def test_index_documents_cleans_text_that_cannot_be_stored(index_path):
    """A UTF-16 client that truncates a title mid-emoji sends half of one, which arrives as
    a lone surrogate. Stored on a page it would cost the whole page, so it is dropped at the
    choke point and the document is indexed like any other."""
    documents = [
        Document(title="Half an emoji \ud83d", url="https://example.com/y", extract="also half \udc00"),
        Document(title="Whole", url="https://example.com/z", extract="a good page"),
    ]

    with patch(PATCH_TARGET, return_value=snapshot_blacklist(set())):
        index_documents(documents, index_path)

    stored = []
    with TinyIndex(item_factory=Document, index_path=index_path, mode="r") as index:
        for page_index in range(10):
            stored += [document for document in index.get_page(page_index) if document.url.endswith("/y")]

    assert stored, "the document was not indexed"
    assert {(document.title, document.extract) for document in stored} == {("Half an emoji ", "also half ")}
    assert "https://example.com/z" in _all_urls(index_path), "the rest of the page was lost"


def test_index_results_against_query_cleans_text_that_cannot_be_stored(index_path):
    """It builds its own documents rather than going through index_documents, so it has to
    clean them itself or lose the page."""
    document = Document(title="Rust \ud83d", url="https://example.com/rust", extract="async \ud83d")

    assert index_results_against_query([document], "rust", index_path) == 1

    with TinyIndex(item_factory=Document, index_path=index_path, mode="r") as index:
        stored = index.retrieve("rust")
    assert [d.title for d in stored] == ["Rust "]