An open source, non-profit web search engine
Never suggest a rejection that cannot be sent back
Daoud Clarkebbdcd18006a5
diff --git a/mwmbl/moderation/model.py b/mwmbl/moderation/model.pyindex dccd9d1..c734cd8 100644--- a/mwmbl/moderation/model.py+++ b/mwmbl/moderation/model.py@@ -37,7 +37,7 @@ from django.utils import timezonefrom mwmbl.models import ModerationModelArtifactfrom mwmbl.moderation.features import Featuriser, ModerationExample-from mwmbl.moderation.rules import APPROVE, EvidenceItem, REJECT, decisive+from mwmbl.moderation.rules import APPROVE, EvidenceItem, REJECT, decisive, implied_detaillogger = getLogger(__name__)@@ -62,11 +62,22 @@ MODEL_REFRESH_SECONDS = 60REASON_CONFIDENCE_CAP = {"OFFENSIVE": 0.6}+# Reasons a suggestion cannot be handed to a moderator as a one-keystroke rejection. OTHER is+# a real class - moderators reach for it often - but it is the one reason that says nothing on+# its own, so the API refuses it without a written detail, and the model has no sentence to+# offer. A blank reason is here for the same reason from the other end: it names nothing at+# all. A check that implies OTHER is unaffected; its label is the detail (see implied_detail).+UNUSABLE_MODEL_REASONS = frozenset({"", "OTHER"})++@dataclassclass Suggestion:action: str # APPROVE | REJECT | UNSUREconfidence: floatreason: str = ""+ # What the submitter would be told. Non-empty whenever ``reason`` is OTHER and empty+ # otherwise: the other reasons are their own explanation.+ reason_detail: str = ""reason_confidence: float = 0.0reason_source: str = "model" # model | derived | rulemodel_version: str = ""@@ -144,6 +155,7 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIteaction=decisive_item.implies_action,confidence=decisive_item.implies_confidence,reason=decisive_item.implies_reason,+ reason_detail=implied_detail(decisive_item),reason_confidence=decisive_item.implies_confidence,reason_source="rule",model_version=model.version if model else "",@@ -179,6 +191,15 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIteaction, confidence = "UNSURE", 1.0 - abs(reject_probability - 0.5) * 2suggested_reason = reason if action == "REJECT" else ""+ if action == "REJECT" and suggested_reason in UNUSABLE_MODEL_REASONS:+ # A rejection we cannot give the reason for is not one to suggest. The reason head can+ # land on OTHER - it is trained on decisions and moderators use it - but nothing here+ # can write the detail that goes with it, and a "Reject - other" button that fails+ # validation when pressed is worse than no suggestion. Back to the moderator, who is+ # the only one who can say what is wrong.+ return Suggestion(action="UNSURE", confidence=0.0, model_version=model.version,+ evidence=[item.to_dict() for item in evidence_items])+return Suggestion(action=action,confidence=float(confidence),diff --git a/mwmbl/moderation/rules.py b/mwmbl/moderation/rules.pyindex 81f953a..8638e2e 100644--- a/mwmbl/moderation/rules.py+++ b/mwmbl/moderation/rules.py@@ -205,6 +205,18 @@ def decisive(items: list[EvidenceItem]) -> Optional[EvidenceItem]:return max(decisive_items, key=lambda item: item.implies_confidence)+def implied_detail(item: EvidenceItem) -> str:+ """What a check's rejection tells the submitter, for a reason that says nothing on its own.++ Only OTHER needs one - "spam" explains itself, "other" does not - and the check's label is+ already that sentence: "Homepage returns HTTP 404" is exactly what was wrong. Every OTHER+ a check implies is one of these, which is what lets the API refuse an OTHER rejection with+ no detail (mwmbl.platform.schemas.RejectionFieldsMixin) without making the suggestion+ untakeable.+ """+ return item.label if item.implies_reason == "OTHER" else ""++def _tld(domain: str) -> str:labels = domain.lower().split(".")return labels[-1] if len(labels) > 1 else ""diff --git a/mwmbl/moderation/suggest.py b/mwmbl/moderation/suggest.pyindex fdd9d34..421625f 100644--- a/mwmbl/moderation/suggest.py+++ b/mwmbl/moderation/suggest.py@@ -31,7 +31,7 @@ from django.db.models.functions import Coalesce, Substrfrom mwmbl.models import DomainEvidence, DomainSubmission, SearchResultVotefrom mwmbl.moderation import rulesfrom mwmbl.moderation.evidence import page_texts-from mwmbl.moderation.model import Suggestion, suggest+from mwmbl.moderation.model import Suggestion, UNUSABLE_MODEL_REASONS, suggestlogger = getLogger(__name__)@@ -87,6 +87,7 @@ def suggestion_for(submission: DomainSubmission,action=decisive_live.implies_action,confidence=decisive_live.implies_confidence,reason=decisive_live.implies_reason,+ reason_detail=rules.implied_detail(decisive_live),reason_confidence=decisive_live.implies_confidence,reason_source="rule",model_version=evidence.model_version,@@ -105,10 +106,25 @@ def suggestion_for(submission: DomainSubmission,# (precision 0.88 on this same slice), so only the approve side is withheld.action, confidence = "UNSURE", 0.0+ reason = evidence.suggested_reason if action == "REJECT" else ""+ source = evidence.reason_source or "model"+ # OTHER is the one reason that explains nothing on its own, and a decision may not carry+ # it without the sentence the submitter is shown. When a check decided it that sentence is+ # the check's own label, which is what reason_source == "rule" means here; when the model+ # did there is none, and a suggestion no client can send back is not one to draw - so it+ # goes to the moderator as UNSURE instead. model.suggest already stops storing those; this+ # is for the rows stored before it did, and annotate_queue mirrors it so the queue filters+ # and orders on what is actually drawn.+ detail = (rules.implied_detail(cached_decisive)+ if reason == "OTHER" and source == "rule" and cached_decisive is not None else "")+ if action == "REJECT" and reason in UNUSABLE_MODEL_REASONS and not detail:+ action, confidence, reason = "UNSURE", 0.0, ""+return Suggestion(action=action,confidence=confidence,- reason=evidence.suggested_reason if action == "REJECT" else "",+ reason=reason,+ reason_detail=detail,# The reason's own confidence, not the rejection's. A model can be sure a domain# should go and barely have a view on why, and showing "SPAM (0.92)" when the reason# head scored 0.31 would present a guess as a finding.@@ -177,11 +193,19 @@ def annotate_queue(submissions):# No cached check implies APPROVE - they are all reasons to reject - so a stored APPROVE# is always the model's, which is what the withheld-approval rule is about.withheld = scored & Q(evidence_action="APPROVE") & Q(submitter_decided=False)+ # suggestion_for's other downgrade: a stored REJECT nobody can send back - one whose+ # reason is OTHER with no check behind it, so there is no detail to send, or one carrying+ # no reason at all. reason_source is again how we know a check produced it, and only then+ # is the detail the check's own label.+ unexplained = scored & Q(evidence_action="REJECT") & (+ (Q(evidence_reason="OTHER") & ~Q(evidence_source="rule"))+ | Q(evidence_reason="") | Q(evidence_reason__isnull=True))submissions = submissions.annotate(displayed_action=Case(When(prior_approval, then=Value("APPROVE")),When(prior_rejection, then=Value("REJECT")),+ When(unexplained, then=Value("UNSURE")),When(withheld, then=Value("UNSURE")),When(scored, then=F("evidence_action")),default=Value(None),@@ -190,6 +214,7 @@ def annotate_queue(submissions):displayed_confidence=Case(When(prior_approval | prior_rejection,then=Value(rules.PRIOR_DECISION_CONFIDENCE)),+ When(unexplained, then=Value(0.0)),When(withheld, then=Value(0.0)),When(scored, then=F("evidence_confidence")),default=Value(None),@@ -197,7 +222,7 @@ def annotate_queue(submissions):),displayed_reason=Case(When(prior_rejection, then=Value(rules.PRIOR_DECISION_REASON)),- When(scored & Q(evidence_action="REJECT") & ~prior_approval,+ When(scored & Q(evidence_action="REJECT") & ~prior_approval & ~unexplained,then=F("evidence_reason")),When(scored, then=Value("")),default=Value(None),diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.pyindex 1dae703..576ea5f 100644--- a/mwmbl/platform/schemas.py+++ b/mwmbl/platform/schemas.py@@ -248,6 +248,14 @@ class SuggestionSchema(Schema):action: str = Field(description="`APPROVE`, `REJECT` or `UNSURE`.")confidence: floatreason: str = Field(default="", description="Rejection reason; empty unless action is REJECT.")+ reason_detail: str = Field(+ default="",+ description=(+ "What the submitter would be told, to be sent back as `rejection_detail`. "+ "Non-empty whenever `reason` is OTHER, which a decision may not carry without "+ "one, and empty otherwise - the other reasons explain themselves."+ ),+ )reason_confidence: float = 0.0reason_source: str = Field(default="model",diff --git a/test/test_domain_moderation.py b/test/test_domain_moderation.pyindex e66a82f..05e73af 100644--- a/test/test_domain_moderation.py+++ b/test/test_domain_moderation.py@@ -89,7 +89,13 @@ def ready_evidence(domain, **overrides):"evidence": [],"model_version": "test-model",}- return DomainEvidence.objects.create(domain=domain, **(defaults | overrides))+ fields = defaults | overrides+ # A stored REJECT always names a reason - the reason head returns one of its classes, and+ # every check that implies REJECT implies OTHER - and suggestion_for will not show one+ # that does not. A fixture without a reason would be a row that cannot occur.+ if fields["suggested_action"] == "REJECT" and not fields.get("suggested_reason"):+ fields["suggested_reason"] = "SPAM"+ return DomainEvidence.objects.create(domain=domain, **fields)# --------------------------------------------------------------------- rules@@ -150,6 +156,40 @@ def test_a_deterministic_check_beats_the_model():model.predict.assert_not_called()+def test_a_check_that_implies_other_carries_the_detail_the_submitter_is_shown():+ """OTHER is the one reason that explains nothing on its own, so a suggestion carrying it+ has to carry the sentence too - the API refuses a decision that does not."""+ items = rules.crawl_evidence(+ "example.com", {"http_status": 404, "pages": [], "signals": {}})+ suggestion = suggest("example.com", [], items, model=mock.Mock(version="test"))++ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")+ assert suggestion.reason_detail == "Homepage returns HTTP 404"+++def test_a_reason_that_explains_itself_needs_no_detail():+ model = mock.Mock(version="test")+ model.predict.return_value = [(0.92, "SPAM", 0.8)]++ suggestion = suggest("example.com", [], [], model=model)++ assert (suggestion.action, suggestion.reason) == ("REJECT", "SPAM")+ assert suggestion.reason_detail == ""++[email protected]("reason", ["OTHER", ""])+def test_the_model_does_not_suggest_a_rejection_it_cannot_give_the_reason_for(reason):+ """The reason head is trained on decisions and moderators do use OTHER, but nothing here+ can write the detail that has to go with it. A "Reject - other" button that fails+ validation when pressed is worse than no suggestion, so it goes back as UNSURE."""+ model = mock.Mock(version="test")+ model.predict.return_value = [(0.92, reason, 0.8)]++ suggestion = suggest("example.com", [], [], model=model)++ assert (suggestion.action, suggestion.reason, suggestion.reason_detail) == ("UNSURE", "", "")++def test_missing_model_degrades_to_unsure_not_to_a_default():with mock.patch("mwmbl.moderation.model.get_model", return_value=None):suggestion = suggest("example.com", ["some text"], [])@@ -1214,6 +1254,50 @@ def test_the_reason_is_shown_with_its_own_confidence(submitter):assert suggestion.reason_source == "model"+def test_a_stored_unexplainable_rejection_is_not_shown_as_a_suggestion(submitter):+ """Rows scored before the model stopped proposing a bare OTHER are still in the table, and+ a moderator pressing the button on one gets a 422 rather than a decision."""+ submission = DomainSubmission.objects.create(name="stale.example", submitted_by=submitter)+ evidence = ready_evidence("stale.example", suggested_action="REJECT",+ suggested_reason="OTHER", confidence=0.93,+ reason_source="model")++ suggestion = suggestion_for(submission, evidence)++ assert (suggestion.action, suggestion.reason, suggestion.reason_detail) == ("UNSURE", "", "")+++def test_a_rule_scored_rejection_keeps_its_reason_and_carries_the_detail(submitter):+ a_404 = rules.EvidenceItem(+ "http_status", rules.REJECT, "Homepage returns HTTP 404", implies_action="REJECT",+ implies_reason="OTHER", implies_confidence=0.9).to_dict()+ submission = DomainSubmission.objects.create(name="dead.example", submitted_by=submitter)+ evidence = ready_evidence("dead.example", suggested_action="REJECT",+ suggested_reason="OTHER", confidence=0.9,+ reason_source="rule", evidence=[a_404])++ suggestion = suggestion_for(submission, evidence)++ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")+ assert suggestion.reason_detail == "Homepage returns HTTP 404"+++def test_a_previous_rejection_of_the_same_domain_says_so_as_the_detail(submitter, established):+ """The prior-decision check implies OTHER too, and its label is the sentence."""+ DomainSubmission.objects.create(name="again.example", submitted_by=established,+ status="REJECTED")+ submission = DomainSubmission.objects.create(name="again.example", submitted_by=submitter)+ evidence = ready_evidence("again.example")++ suggestion = suggestion_for(submission, evidence)++ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")+ assert suggestion.reason_detail == "This domain has already been rejected before"++# --------------------------------------------------------------------- queue parity@pytest.mark.django_db@@ -1239,6 +1323,9 @@ def test_queue_display_matches_suggestion_for(submitter, established):("reject.example", submitter, {"suggested_action": "REJECT","suggested_reason": "SPAM", "confidence": 0.91}),("unsure.example", submitter, {"suggested_action": "UNSURE", "confidence": 0.1}),+ # A reason the model cannot explain, so not shown as a suggestion at all.+ ("unexplained.example", submitter, {"suggested_action": "REJECT",+ "suggested_reason": "OTHER", "confidence": 0.93}),("approved-before.example", submitter, {"suggested_action": "REJECT","suggested_reason": "SPAM"}),("rejected-before.example", submitter, {}),