Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

7.0 KB
Raw Blame
from datetime import timedelta

import fakeredis
import pytest
from django.contrib.auth import get_user_model
from django.test import Client

from mwmbl.crawler.worst_agreements import (
    STORED,
    SUBMITTER,
    UNKNOWN_GAP,
    WORST_PER_BUCKET,
    agreement_score,
    gap_bucket,
    read_worst_agreements,
    record_worst_agreements,
)
from mwmbl.indexer.agreement import Agreement
from mwmbl.search_callers import window_start
from mwmbl.utils import utc_today

SUBMITTER_ID = 1
STORED_ID = 2


@pytest.fixture
def client():
    return fakeredis.FakeRedis(decode_responses=True)


def _today():
    """Not a fixed date: the keys expire a few weeks after the week they are for."""
    return utc_today()


def _window():
    return window_start(_today())


def _agreement(
    url="https://example.com/",
    title_similarity=1.0,
    extract_similarity=0.5,
    delta=60,
    extract="Spam",
    existing_extract="Real",
    existing_user_id=STORED_ID,
):
    return Agreement(
        url=url,
        existing_user_id=existing_user_id,
        title_similarity=title_similarity,
        extract_similarity=extract_similarity,
        extract_empty=not extract.strip(),
        existing_extract_empty=not existing_extract.strip(),
        last_crawled_delta=delta,
        title="Title",
        extract=extract,
        existing_title="Title",
        existing_extract=existing_extract,
    )


def _record(client, agreements, user_id=SUBMITTER_ID):
    record_worst_agreements(client, agreements, user_id, _today())


def _read(client, user_id, role, bucket="hour"):
    return read_worst_agreements(client, user_id, _window())[role][bucket]


@pytest.mark.parametrize(
    "delta, bucket",
    [
        (None, UNKNOWN_GAP.name),
        (0, "hour"),
        (-3599, "hour"),
        (3600, "day"),
        (-86400 * 3, "week"),
        (86400 * 30, "older"),
    ],
)
def test_gap_bucket(delta, bucket):
    assert gap_bucket(delta).name == bucket


def test_agreement_score_takes_the_lower_similarity():
    assert agreement_score(_agreement(title_similarity=0.2, extract_similarity=0.9)) == 0.2
    assert agreement_score(_agreement(title_similarity=0.9, extract_similarity=0.3)) == 0.3


def test_agreement_score_compares_titles_alone_when_an_extract_is_empty():
    assert agreement_score(_agreement(title_similarity=0.7, extract_similarity=0.0, extract="")) == 0.7


def test_keeps_the_comparison_under_both_crawlers(client):
    _record(client, [_agreement()])

    (submitted,) = _read(client, SUBMITTER_ID, SUBMITTER)
    (stored,) = _read(client, STORED_ID, STORED)

    assert submitted["other_user_id"] == STORED_ID
    assert (submitted["own"]["extract"], submitted["other"]["extract"]) == ("Spam", "Real")
    assert stored["other_user_id"] == SUBMITTER_ID
    assert (stored["own"]["extract"], stored["other"]["extract"]) == ("Real", "Spam")
    assert _read(client, SUBMITTER_ID, STORED) == []
    assert _read(client, STORED_ID, SUBMITTER) == []


def test_keeps_only_the_least_agreeing(client):
    agreements = [
        _agreement(url=f"https://example.com/{i}", extract_similarity=i / 100) for i in range(WORST_PER_BUCKET + 10)
    ]

    _record(client, agreements)

    kept = _read(client, SUBMITTER_ID, SUBMITTER)
    assert [entry["score"] for entry in kept] == [i / 100 for i in range(WORST_PER_BUCKET)]
    assert client.hlen(next(iter(client.keys("*:submitter:*:entries")))) == WORST_PER_BUCKET


def test_ranks_each_crawl_gap_separately(client):
    """A stale copy scoring lower does not push out a fresh one."""
    stale = [_agreement(url=f"https://example.com/{i}", extract_similarity=0.1, delta=86400 * 30) for i in range(60)]
    fresh = _agreement(url="https://example.com/fresh", extract_similarity=0.6, delta=60)

    _record(client, stale + [fresh])

    assert [entry["url"] for entry in _read(client, SUBMITTER_ID, SUBMITTER)] == ["https://example.com/fresh"]
    assert len(_read(client, SUBMITTER_ID, SUBMITTER, "older")) == WORST_PER_BUCKET


def test_other_crawlers_cannot_push_out_a_crawlers_own_submissions(client):
    _record(client, [_agreement(url="https://example.com/spam", extract_similarity=0.3)], user_id=SUBMITTER_ID)
    junk = [
        _agreement(url=f"https://example.com/{i}", extract_similarity=0.01, existing_user_id=SUBMITTER_ID)
        for i in range(WORST_PER_BUCKET)
    ]
    _record(client, junk, user_id=3)

    assert [entry["url"] for entry in _read(client, SUBMITTER_ID, SUBMITTER)] == ["https://example.com/spam"]
    assert len(_read(client, SUBMITTER_ID, STORED)) == WORST_PER_BUCKET


def test_a_url_compared_again_replaces_its_entry(client):
    for delta in range(60):
        _record(client, [_agreement(delta=delta, extract=f"Spam {delta}")])

    (kept,) = _read(client, SUBMITTER_ID, SUBMITTER)
    assert kept["own"]["extract"] == "Spam 59"


def test_keeps_empty_extracts_ranked_by_title(client):
    _record(client, [_agreement(title_similarity=0.4, extract="")])

    (kept,) = _read(client, SUBMITTER_ID, SUBMITTER)
    assert kept["score"] == 0.4


def test_keeps_each_week_apart(client):
    _record(client, [_agreement()])

    previous_week = read_worst_agreements(client, SUBMITTER_ID, _window() - timedelta(weeks=1))
    assert not any(entries for by_bucket in previous_week.values() for entries in by_bucket.values())


def test_keys_expire(client):
    _record(client, [_agreement()])

    keys = client.keys("crawler:worst-agreements:*")
    assert keys
    assert all(client.ttl(key) > 0 for key in keys)


@pytest.fixture
def admin_browser(db):
    admin = get_user_model().objects.create_superuser(username="admin", email="[email protected]", password="pw")
    browser = Client()
    browser.force_login(admin)
    return browser


@pytest.mark.django_db
def test_admin_page_shows_both_copies(client, admin_browser, monkeypatch):
    User = get_user_model()
    submitter = User.objects.create_user(username="sneaky", email="[email protected]", password="pw")
    other = User.objects.create_user(username="honest", email="[email protected]", password="pw")
    agreement = _agreement(extract="Buy cheap pills", existing_extract="Learn python", existing_user_id=other.id)
    _record(client, [agreement], user_id=submitter.id)
    monkeypatch.setattr("mwmbl.crawler.worst_agreements.get_redis", lambda: client)

    response = admin_browser.get("/admin/worst-agreements/", {"username": "sneaky"})

    content = response.content.decode()
    assert response.status_code == 200
    assert "Buy cheap pills" in content
    assert "Learn python" in content
    assert "sneaky submitted over honest's copy" in content


@pytest.mark.django_db
def test_admin_page_with_an_unknown_user(admin_browser):
    response = admin_browser.get("/admin/worst-agreements/", {"username": "nobody"})

    assert "No user named nobody" in response.content.decode()


@pytest.mark.django_db
@pytest.mark.parametrize("weeks_ago", ["abc", "-1", "4"])
def test_admin_page_rejects_a_week_it_does_not_keep(admin_browser, weeks_ago):
    response = admin_browser.get("/admin/worst-agreements/", {"username": "nobody", "weeks_ago": weeks_ago})

    assert response.status_code == 400