Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Never suggest a rejection that cannot be sent back

Daoud Clarkebbdcd18006a5
diff --git a/mwmbl/moderation/model.py b/mwmbl/moderation/model.py
index dccd9d1..c734cd8 100644
--- a/mwmbl/moderation/model.py
+++ b/mwmbl/moderation/model.py
@@ -37,7 +37,7 @@ from django.utils import timezone
from mwmbl.models import ModerationModelArtifact
from mwmbl.moderation.features import Featuriser, ModerationExample
-from mwmbl.moderation.rules import APPROVE, EvidenceItem, REJECT, decisive
+from mwmbl.moderation.rules import APPROVE, EvidenceItem, REJECT, decisive, implied_detail
logger = getLogger(__name__)
@@ -62,11 +62,22 @@ MODEL_REFRESH_SECONDS = 60
REASON_CONFIDENCE_CAP = {"OFFENSIVE": 0.6}
+# Reasons a suggestion cannot be handed to a moderator as a one-keystroke rejection. OTHER is
+# a real class - moderators reach for it often - but it is the one reason that says nothing on
+# its own, so the API refuses it without a written detail, and the model has no sentence to
+# offer. A blank reason is here for the same reason from the other end: it names nothing at
+# all. A check that implies OTHER is unaffected; its label is the detail (see implied_detail).
+UNUSABLE_MODEL_REASONS = frozenset({"", "OTHER"})
+
+
@dataclass
class Suggestion:
action: str # APPROVE | REJECT | UNSURE
confidence: float
reason: str = ""
+ # What the submitter would be told. Non-empty whenever ``reason`` is OTHER and empty
+ # otherwise: the other reasons are their own explanation.
+ reason_detail: str = ""
reason_confidence: float = 0.0
reason_source: str = "model" # model | derived | rule
model_version: str = ""
@@ -144,6 +155,7 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIte
action=decisive_item.implies_action,
confidence=decisive_item.implies_confidence,
reason=decisive_item.implies_reason,
+ reason_detail=implied_detail(decisive_item),
reason_confidence=decisive_item.implies_confidence,
reason_source="rule",
model_version=model.version if model else "",
@@ -179,6 +191,15 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIte
action, confidence = "UNSURE", 1.0 - abs(reject_probability - 0.5) * 2
suggested_reason = reason if action == "REJECT" else ""
+ if action == "REJECT" and suggested_reason in UNUSABLE_MODEL_REASONS:
+ # A rejection we cannot give the reason for is not one to suggest. The reason head can
+ # land on OTHER - it is trained on decisions and moderators use it - but nothing here
+ # can write the detail that goes with it, and a "Reject - other" button that fails
+ # validation when pressed is worse than no suggestion. Back to the moderator, who is
+ # the only one who can say what is wrong.
+ return Suggestion(action="UNSURE", confidence=0.0, model_version=model.version,
+ evidence=[item.to_dict() for item in evidence_items])
+
return Suggestion(
action=action,
confidence=float(confidence),
diff --git a/mwmbl/moderation/rules.py b/mwmbl/moderation/rules.py
index 81f953a..8638e2e 100644
--- a/mwmbl/moderation/rules.py
+++ b/mwmbl/moderation/rules.py
@@ -205,6 +205,18 @@ def decisive(items: list[EvidenceItem]) -> Optional[EvidenceItem]:
return max(decisive_items, key=lambda item: item.implies_confidence)
+def implied_detail(item: EvidenceItem) -> str:
+ """What a check's rejection tells the submitter, for a reason that says nothing on its own.
+
+ Only OTHER needs one - "spam" explains itself, "other" does not - and the check's label is
+ already that sentence: "Homepage returns HTTP 404" is exactly what was wrong. Every OTHER
+ a check implies is one of these, which is what lets the API refuse an OTHER rejection with
+ no detail (mwmbl.platform.schemas.RejectionFieldsMixin) without making the suggestion
+ untakeable.
+ """
+ return item.label if item.implies_reason == "OTHER" else ""
+
+
def _tld(domain: str) -> str:
labels = domain.lower().split(".")
return labels[-1] if len(labels) > 1 else ""
diff --git a/mwmbl/moderation/suggest.py b/mwmbl/moderation/suggest.py
index fdd9d34..421625f 100644
--- a/mwmbl/moderation/suggest.py
+++ b/mwmbl/moderation/suggest.py
@@ -31,7 +31,7 @@ from django.db.models.functions import Coalesce, Substr
from mwmbl.models import DomainEvidence, DomainSubmission, SearchResultVote
from mwmbl.moderation import rules
from mwmbl.moderation.evidence import page_texts
-from mwmbl.moderation.model import Suggestion, suggest
+from mwmbl.moderation.model import Suggestion, UNUSABLE_MODEL_REASONS, suggest
logger = getLogger(__name__)
@@ -87,6 +87,7 @@ def suggestion_for(submission: DomainSubmission,
action=decisive_live.implies_action,
confidence=decisive_live.implies_confidence,
reason=decisive_live.implies_reason,
+ reason_detail=rules.implied_detail(decisive_live),
reason_confidence=decisive_live.implies_confidence,
reason_source="rule",
model_version=evidence.model_version,
@@ -105,10 +106,25 @@ def suggestion_for(submission: DomainSubmission,
# (precision 0.88 on this same slice), so only the approve side is withheld.
action, confidence = "UNSURE", 0.0
+ reason = evidence.suggested_reason if action == "REJECT" else ""
+ source = evidence.reason_source or "model"
+ # OTHER is the one reason that explains nothing on its own, and a decision may not carry
+ # it without the sentence the submitter is shown. When a check decided it that sentence is
+ # the check's own label, which is what reason_source == "rule" means here; when the model
+ # did there is none, and a suggestion no client can send back is not one to draw - so it
+ # goes to the moderator as UNSURE instead. model.suggest already stops storing those; this
+ # is for the rows stored before it did, and annotate_queue mirrors it so the queue filters
+ # and orders on what is actually drawn.
+ detail = (rules.implied_detail(cached_decisive)
+ if reason == "OTHER" and source == "rule" and cached_decisive is not None else "")
+ if action == "REJECT" and reason in UNUSABLE_MODEL_REASONS and not detail:
+ action, confidence, reason = "UNSURE", 0.0, ""
+
return Suggestion(
action=action,
confidence=confidence,
- reason=evidence.suggested_reason if action == "REJECT" else "",
+ reason=reason,
+ reason_detail=detail,
# The reason's own confidence, not the rejection's. A model can be sure a domain
# should go and barely have a view on why, and showing "SPAM (0.92)" when the reason
# head scored 0.31 would present a guess as a finding.
@@ -177,11 +193,19 @@ def annotate_queue(submissions):
# No cached check implies APPROVE - they are all reasons to reject - so a stored APPROVE
# is always the model's, which is what the withheld-approval rule is about.
withheld = scored & Q(evidence_action="APPROVE") & Q(submitter_decided=False)
+ # suggestion_for's other downgrade: a stored REJECT nobody can send back - one whose
+ # reason is OTHER with no check behind it, so there is no detail to send, or one carrying
+ # no reason at all. reason_source is again how we know a check produced it, and only then
+ # is the detail the check's own label.
+ unexplained = scored & Q(evidence_action="REJECT") & (
+ (Q(evidence_reason="OTHER") & ~Q(evidence_source="rule"))
+ | Q(evidence_reason="") | Q(evidence_reason__isnull=True))
submissions = submissions.annotate(
displayed_action=Case(
When(prior_approval, then=Value("APPROVE")),
When(prior_rejection, then=Value("REJECT")),
+ When(unexplained, then=Value("UNSURE")),
When(withheld, then=Value("UNSURE")),
When(scored, then=F("evidence_action")),
default=Value(None),
@@ -190,6 +214,7 @@ def annotate_queue(submissions):
displayed_confidence=Case(
When(prior_approval | prior_rejection,
then=Value(rules.PRIOR_DECISION_CONFIDENCE)),
+ When(unexplained, then=Value(0.0)),
When(withheld, then=Value(0.0)),
When(scored, then=F("evidence_confidence")),
default=Value(None),
@@ -197,7 +222,7 @@ def annotate_queue(submissions):
),
displayed_reason=Case(
When(prior_rejection, then=Value(rules.PRIOR_DECISION_REASON)),
- When(scored & Q(evidence_action="REJECT") & ~prior_approval,
+ When(scored & Q(evidence_action="REJECT") & ~prior_approval & ~unexplained,
then=F("evidence_reason")),
When(scored, then=Value("")),
default=Value(None),
diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.py
index 1dae703..576ea5f 100644
--- a/mwmbl/platform/schemas.py
+++ b/mwmbl/platform/schemas.py
@@ -248,6 +248,14 @@ class SuggestionSchema(Schema):
action: str = Field(description="`APPROVE`, `REJECT` or `UNSURE`.")
confidence: float
reason: str = Field(default="", description="Rejection reason; empty unless action is REJECT.")
+ reason_detail: str = Field(
+ default="",
+ description=(
+ "What the submitter would be told, to be sent back as `rejection_detail`. "
+ "Non-empty whenever `reason` is OTHER, which a decision may not carry without "
+ "one, and empty otherwise - the other reasons explain themselves."
+ ),
+ )
reason_confidence: float = 0.0
reason_source: str = Field(
default="model",
diff --git a/test/test_domain_moderation.py b/test/test_domain_moderation.py
index e66a82f..05e73af 100644
--- a/test/test_domain_moderation.py
+++ b/test/test_domain_moderation.py
@@ -89,7 +89,13 @@ def ready_evidence(domain, **overrides):
"evidence": [],
"model_version": "test-model",
}
- return DomainEvidence.objects.create(domain=domain, **(defaults | overrides))
+ fields = defaults | overrides
+ # A stored REJECT always names a reason - the reason head returns one of its classes, and
+ # every check that implies REJECT implies OTHER - and suggestion_for will not show one
+ # that does not. A fixture without a reason would be a row that cannot occur.
+ if fields["suggested_action"] == "REJECT" and not fields.get("suggested_reason"):
+ fields["suggested_reason"] = "SPAM"
+ return DomainEvidence.objects.create(domain=domain, **fields)
# --------------------------------------------------------------------- rules
@@ -150,6 +156,40 @@ def test_a_deterministic_check_beats_the_model():
model.predict.assert_not_called()
+def test_a_check_that_implies_other_carries_the_detail_the_submitter_is_shown():
+ """OTHER is the one reason that explains nothing on its own, so a suggestion carrying it
+ has to carry the sentence too - the API refuses a decision that does not."""
+ items = rules.crawl_evidence(
+ "example.com", {"http_status": 404, "pages": [], "signals": {}})
+ suggestion = suggest("example.com", [], items, model=mock.Mock(version="test"))
+
+ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")
+ assert suggestion.reason_detail == "Homepage returns HTTP 404"
+
+
+def test_a_reason_that_explains_itself_needs_no_detail():
+ model = mock.Mock(version="test")
+ model.predict.return_value = [(0.92, "SPAM", 0.8)]
+
+ suggestion = suggest("example.com", [], [], model=model)
+
+ assert (suggestion.action, suggestion.reason) == ("REJECT", "SPAM")
+ assert suggestion.reason_detail == ""
+
+
[email protected]("reason", ["OTHER", ""])
+def test_the_model_does_not_suggest_a_rejection_it_cannot_give_the_reason_for(reason):
+ """The reason head is trained on decisions and moderators do use OTHER, but nothing here
+ can write the detail that has to go with it. A "Reject - other" button that fails
+ validation when pressed is worse than no suggestion, so it goes back as UNSURE."""
+ model = mock.Mock(version="test")
+ model.predict.return_value = [(0.92, reason, 0.8)]
+
+ suggestion = suggest("example.com", [], [], model=model)
+
+ assert (suggestion.action, suggestion.reason, suggestion.reason_detail) == ("UNSURE", "", "")
+
+
def test_missing_model_degrades_to_unsure_not_to_a_default():
with mock.patch("mwmbl.moderation.model.get_model", return_value=None):
suggestion = suggest("example.com", ["some text"], [])
@@ -1214,6 +1254,50 @@ def test_the_reason_is_shown_with_its_own_confidence(submitter):
assert suggestion.reason_source == "model"
+def test_a_stored_unexplainable_rejection_is_not_shown_as_a_suggestion(submitter):
+ """Rows scored before the model stopped proposing a bare OTHER are still in the table, and
+ a moderator pressing the button on one gets a 422 rather than a decision."""
+ submission = DomainSubmission.objects.create(name="stale.example", submitted_by=submitter)
+ evidence = ready_evidence("stale.example", suggested_action="REJECT",
+ suggested_reason="OTHER", confidence=0.93,
+ reason_source="model")
+
+ suggestion = suggestion_for(submission, evidence)
+
+ assert (suggestion.action, suggestion.reason, suggestion.reason_detail) == ("UNSURE", "", "")
+
+
+def test_a_rule_scored_rejection_keeps_its_reason_and_carries_the_detail(submitter):
+ a_404 = rules.EvidenceItem(
+ "http_status", rules.REJECT, "Homepage returns HTTP 404", implies_action="REJECT",
+ implies_reason="OTHER", implies_confidence=0.9).to_dict()
+ submission = DomainSubmission.objects.create(name="dead.example", submitted_by=submitter)
+ evidence = ready_evidence("dead.example", suggested_action="REJECT",
+ suggested_reason="OTHER", confidence=0.9,
+ reason_source="rule", evidence=[a_404])
+
+ suggestion = suggestion_for(submission, evidence)
+
+ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")
+ assert suggestion.reason_detail == "Homepage returns HTTP 404"
+
+
+def test_a_previous_rejection_of_the_same_domain_says_so_as_the_detail(submitter, established):
+ """The prior-decision check implies OTHER too, and its label is the sentence."""
+ DomainSubmission.objects.create(name="again.example", submitted_by=established,
+ status="REJECTED")
+ submission = DomainSubmission.objects.create(name="again.example", submitted_by=submitter)
+ evidence = ready_evidence("again.example")
+
+ suggestion = suggestion_for(submission, evidence)
+
+ assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")
+ assert suggestion.reason_detail == "This domain has already been rejected before"
+
+
# --------------------------------------------------------------------- queue parity
@pytest.mark.django_db
@@ -1239,6 +1323,9 @@ def test_queue_display_matches_suggestion_for(submitter, established):
("reject.example", submitter, {"suggested_action": "REJECT",
"suggested_reason": "SPAM", "confidence": 0.91}),
("unsure.example", submitter, {"suggested_action": "UNSURE", "confidence": 0.1}),
+ # A reason the model cannot explain, so not shown as a suggestion at all.
+ ("unexplained.example", submitter, {"suggested_action": "REJECT",
+ "suggested_reason": "OTHER", "confidence": 0.93}),
("approved-before.example", submitter, {"suggested_action": "REJECT",
"suggested_reason": "SPAM"}),
("rejected-before.example", submitter, {}),