Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Stop the retrain gate comparing PR-AUC across different populations

Daoud Clarke24ec37942d15
diff --git a/.gitignore b/.gitignore
index d8fcb86..42bc25e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -216,3 +216,4 @@ devdata/judgments_export/
devdata/judge_train/*
!devdata/judge_train/RESULTS.md
!devdata/judge_train/eval_manifest.json
+devdata/moderation_bakeoff/
diff --git a/mwmbl/management/commands/train_domain_moderation_model.py b/mwmbl/management/commands/train_domain_moderation_model.py
index 92a2c94..961afe4 100644
--- a/mwmbl/management/commands/train_domain_moderation_model.py
+++ b/mwmbl/management/commands/train_domain_moderation_model.py
@@ -3,8 +3,14 @@
uv run manage.py train_domain_moderation_model --dry-run
uv run manage.py train_domain_moderation_model
-The gate compares the new model's cold-start PR-AUC against the lower bound of the incumbent's
-bootstrap interval, so a change that only looks like an improvement does not ship.
+The gate scores the incumbent and the candidate on the *same* held-out rows and bootstraps the
+difference, so a change that only looks like an improvement does not ship - and, just as
+importantly, a change that only looks like a regression because the population moved underneath
+it is not blocked. See mwmbl.moderation.train for why raw PR-AUC cannot be compared across two
+runs.
+
+ # is the crawled page text earning its place?
+ uv run manage.py train_domain_moderation_model --dry-run --ablate
Publishing writes a ModerationModelArtifact row rather than a file: the incumbent the gate
reads and the model the workers serve are then the same object, and neither is lost to a
@@ -18,7 +24,7 @@ from django.core.management.base import BaseCommand
from mwmbl.background import rescore_pending_submissions
from mwmbl.models import DomainSubmission
from mwmbl.moderation import training_data
-from mwmbl.moderation.model import load_metrics, publish
+from mwmbl.moderation.model import load_metrics, load_published_model, publish
from mwmbl.moderation.train import passes_gate, train
# A reason class with fewer real examples than this is a candidate for blocklist-derived
@@ -37,6 +43,13 @@ class Command(BaseCommand):
help="Train and report metrics without writing the artifact")
parser.add_argument("--force", action="store_true",
help="Publish even if the gate fails")
+ parser.add_argument(
+ "--no-text", action="store_true", dest="no_text",
+ help="Fit on the domain name alone, ignoring crawled page text")
+ parser.add_argument(
+ "--ablate", action="store_true",
+ help=("Also train the domain-name-only model and report both, so the contribution "
+ "of the crawled page text is measured on the same split rather than guessed"))
parser.add_argument(
"--derived", action="store_true",
help=("Add blocklist-derived rows for reason classes short of real data. Off by "
@@ -61,9 +74,17 @@ class Command(BaseCommand):
exclude={row.domain for row in rows})
version = f"domain-mod-{date.today():%Y-%m-%d}"
- model, metrics = train(rows, version)
+ # The model the workers are serving right now, so the gate can score it on the same
+ # held-out rows as the candidate. None on a first run, and on an artifact this code can
+ # no longer featurise - the gate falls back to the stored metrics and says so.
+ incumbent = load_published_model()
+ model, metrics = train(rows, version, use_text=not options["no_text"],
+ incumbent=incumbent)
self.stdout.write(json.dumps(metrics, indent=2))
+ if options["ablate"]:
+ self._report_ablation(rows, version, metrics)
+
allowed, explanation = passes_gate(metrics, load_metrics())
self.stdout.write(("PASS: " if allowed else "FAIL: ") + explanation)
@@ -80,6 +101,31 @@ class Command(BaseCommand):
rescore_pending_submissions(schedule=0)
self.stdout.write("Scheduled a rescore of the pending queue with the new model.")
+ def _report_ablation(self, rows, version, with_text: dict) -> None:
+ """Train the same model without the text block and print both cold-start scores.
+
+ Same rows, same split, same seed, so the difference is the text block and nothing else.
+ Normalised AP rather than PR-AUC because the two fits see identical prevalence here and
+ it keeps the number on the same scale as everything else the retrain reports.
+ """
+ _, without_text = train(rows, f"{version}-no-text", use_text=False)
+ self.stdout.write("\nAblation - does the crawled page text help?")
+ for name, metrics in (("with text", with_text), ("domain name only", without_text)):
+ cold = metrics.get("cold_start")
+ if cold is None:
+ self.stdout.write(f" {name:>16}: no cold-start slice")
+ continue
+ self.stdout.write(
+ f" {name:>16}: normalised AP {cold['normalised_ap']:.4f} "
+ f"[{cold['normalised_ap_ci'][0]:.4f}, {cold['normalised_ap_ci'][1]:.4f}] "
+ f"(PR-AUC {cold['pr_auc']:.4f}, base rate {cold['base_rate']:.4f})")
+ self.stdout.write(
+ f" {with_text.get('train_rows_with_text', 0)} of "
+ f"{sum(with_text.get('train_rows_by_source', {}).values())} training rows and "
+ f"{with_text.get('cold_start', {}).get('rows_with_text', 0)} of "
+ f"{with_text.get('cold_start', {}).get('rows', 0)} cold-start test rows have text. "
+ f"A large gap between those two is the skew to fix before reading the ablation.\n")
+
@staticmethod
def _reasons_needing_data(rows) -> set[str]:
counts: dict[str, int] = {}
diff --git a/mwmbl/moderation/artifacts/README.md b/mwmbl/moderation/artifacts/README.md
index 22df4fc..c6bb5a2 100644
--- a/mwmbl/moderation/artifacts/README.md
+++ b/mwmbl/moderation/artifacts/README.md
@@ -17,24 +17,37 @@ worker replica keeping its own copy.
uv run manage.py train_domain_moderation_model --dry-run # report metrics only
uv run manage.py train_domain_moderation_model # publish if it passes the gate
-The gate compares the new model's cold-start PR-AUC against the *lower bound* of the incumbent's
-bootstrap interval — the metrics stored beside the published model, falling back to `metrics.json`
-until something has been published — so a change that only looks like an improvement does not
-ship. Publishing writes a new artifact row, which every worker picks up within a minute, and
-schedules `rescore_pending_submissions` to re-score the existing queue with the new model.
+The gate loads the published model, scores it and the candidate on the *same* held-out rows, and
+bootstraps the difference; the candidate ships unless it is confidently worse. Raw PR-AUC is never
+compared across two runs, because its floor is the positive rate — the August 2026 retrain was
+blocked for a 0.11 "regression" that was entirely a shift in prevalence after migration 0037. See
+`mwmbl.moderation.train`. Publishing writes a new artifact row, which every worker picks up within
+a minute, and schedules `rescore_pending_submissions` to re-score the existing queue.
To reproduce the numbers without a production database, train from the sanitized judgments export
instead:
uv run python scripts/moderation_eval.py
+ uv run python scripts/moderation_eval.py --write-artifact # overwrite the files here
+
+**Run `--write-artifact` after any change to the feature set.** A featuriser is pickled whole, so
+adding or moving a feature leaves this artifact describing a matrix the code no longer builds. It
+then fails the probe prediction that `mwmbl.moderation.model` makes at load time and is dropped,
+and every deploy runs on the deterministic checks alone until the first retrain publishes. The
+probe is what turns that from an exception per domain inside the enrichment task into an honest
+"not assessed yet" — it is not a licence to leave the artifact stale.
## What this version was trained on
-The committed artifact comes from the judgments export, which carries no page text, so its
-featuriser has **no text block** — it judges on the domain name, TLD and shape alone. Once
-`backfill_domain_evidence` has crawled real submissions, a retrain picks up the titles and
-extracts of the three crawled pages and the text block appears. Expect the first post-backfill
-retrain to change the artifact substantially.
+The committed artifact comes from the judgments export, which carries no page text. Only the 21
+hand-written seed rows have any, so the text block exists but is fitted on those alone and
+contributes almost nothing: in practice this model judges on the domain name, TLD and shape, and
+`has_text` is 0 for every real row it saw. Once `backfill_domain_evidence` has crawled real
+submissions, a retrain picks up the titles and extracts of the three crawled pages and the text
+block becomes real. Expect the first post-backfill retrain to change the artifact substantially,
+and read `train_rows_with_text` against the per-slice `rows_with_text` before believing anything
+about whether the text is helping — evidence is crawled newest-first, so a chronological split
+puts most of the text on the test side.
## Format
diff --git a/mwmbl/moderation/artifacts/metrics.json b/mwmbl/moderation/artifacts/metrics.json
index bb35c53..6aa6ca2 100644
--- a/mwmbl/moderation/artifacts/metrics.json
+++ b/mwmbl/moderation/artifacts/metrics.json
@@ -3,30 +3,44 @@
"slice": "all",
"rows": 949,
"positives": 89,
- "pr_auc": 0.3987,
+ "base_rate": 0.0938,
+ "pr_auc": 0.4033,
"pr_auc_ci": [
- 0.3007,
- 0.4949
+ 0.304,
+ 0.4969
],
- "recall_at_precision_75": 0.2135
+ "normalised_ap": 0.3415,
+ "normalised_ap_ci": [
+ 0.2357,
+ 0.4431
+ ],
+ "recall_at_precision_75": 0.2022,
+ "rows_with_text": 0
},
"cold_start": {
"slice": "cold_start",
"rows": 150,
"positives": 81,
- "pr_auc": 0.7788,
+ "base_rate": 0.54,
+ "pr_auc": 0.7793,
"pr_auc_ci": [
- 0.6885,
- 0.8588
+ 0.6889,
+ 0.8594
+ ],
+ "normalised_ap": 0.5203,
+ "normalised_ap_ci": [
+ 0.3545,
+ 0.6621
],
- "recall_at_precision_75": 0.4074
+ "recall_at_precision_75": 0.4074,
+ "rows_with_text": 0
},
"reason_head": {
"rows": 71,
"f1": {
"LANGUAGE": 0.0,
- "OTHER": 0.3448,
- "SPAM": 0.8288
+ "OTHER": 0.2609,
+ "SPAM": 0.8547
},
"support": {
"LANGUAGE": 2,
@@ -37,5 +51,13 @@
"train_rows_by_source": {
"real": 2847,
"seed": 21
+ },
+ "train_rows_with_text": 21,
+ "uses_text": true,
+ "suggestion_influence": {
+ "real_rows": 3796,
+ "shown_a_suggestion": 0,
+ "suggested_a_side": 0,
+ "agreed_with_it": 0
}
}
diff --git a/mwmbl/moderation/artifacts/model.joblib b/mwmbl/moderation/artifacts/model.joblib
index 0bc1e24..5bfa669 100644
Binary files a/mwmbl/moderation/artifacts/model.joblib and b/mwmbl/moderation/artifacts/model.joblib differ
diff --git a/mwmbl/moderation/features.py b/mwmbl/moderation/features.py
index 6d136ff..d2c6edf 100644
--- a/mwmbl/moderation/features.py
+++ b/mwmbl/moderation/features.py
@@ -32,10 +32,14 @@ SPAM_WORDS = [
# crawler (65 and 155 chars), so three pages is a few hundred characters at most.
MAX_TEXT_CHARS = 2000
-NUM_SHAPE_FEATURES = 8
-# Shape features are raw counts; scaling them into roughly [0, 1] keeps the regularisation
-# comparable across them and the TF-IDF blocks.
+# Counts are scaled into roughly [0, 1] so the regularisation is comparable across them and
+# the TF-IDF blocks. Indicators are already in [0, 1] and are deliberately *not* scaled: a 0/1
+# divided by 40 forces its coefficient forty times larger to say the same thing, which the L2
+# penalty then charges for, and has_text is the one feature that most needs to be heard.
SHAPE_SCALE = 40.0
+COUNT_FEATURE_NAMES = ["len", "labels", "digits", "hyphens", "sld_len", "spamwords", "tld_len"]
+INDICATOR_FEATURE_NAMES = ["www", "has_text"]
+NUM_SHAPE_FEATURES = len(COUNT_FEATURE_NAMES) + len(INDICATOR_FEATURE_NAMES)
@dataclass
@@ -59,6 +63,7 @@ def _tld_tokens(domain: str) -> list[str]:
def shape_features(domain: str) -> list[float]:
+ """The scaled counts, in COUNT_FEATURE_NAMES order."""
host = domain.lower()
labels = host.split(".")
return [
@@ -66,13 +71,27 @@ def shape_features(domain: str) -> list[float]:
len(labels),
sum(character.isdigit() for character in host),
host.count("-"),
- float(host.startswith("www.")),
len(labels[0]),
sum(word in host for word in SPAM_WORDS),
len(tld(domain)),
]
+def indicator_features(example: "ModerationExample") -> list[float]:
+ """The unscaled 0/1 features, in INDICATOR_FEATURE_NAMES order.
+
+ ``has_text`` exists because an empty text block and a page whose words are all out of
+ vocabulary produce the same all-zero TF-IDF row, and they are not the same thing. Evidence
+ is crawled for recent submissions first, so in a chronological split most training rows
+ have no text and most test rows do; without this the model cannot tell the two apart, and
+ the difference is silently absorbed into the intercept it fitted on the training mix.
+ """
+ return [
+ float(example.domain.lower().startswith("www.")),
+ float(bool(example.text.strip())),
+ ]
+
+
class Featuriser:
"""Fits the vectorisers on a training set, then transforms anything the same way.
@@ -80,7 +99,10 @@ class Featuriser:
training saw rather than rebuilding one from different data.
"""
- def __init__(self):
+ def __init__(self, use_text: bool = True):
+ # Set False to fit the domain-name-only model, so the ablation that answers "is the
+ # crawled page text helping?" runs through exactly this code rather than a copy of it.
+ self.use_text = use_text
# char_wb over the domain: catches the morphology of generated spam names, and the
# word-boundary variant keeps n-grams from spanning label boundaries.
self.domain_chars = TfidfVectorizer(
@@ -106,10 +128,12 @@ class Featuriser:
# raises on - so a text block is only added when the text can actually support one.
texts = [example.text for example in examples]
try:
+ if not self.use_text:
+ raise ValueError("text block disabled for this fit")
blocks.append(self.text.fit_transform(texts))
except ValueError:
- logger.info("Not enough page text to fit a text block over %d examples; "
- "the model will judge on the domain name alone", len(examples))
+ logger.info("No text block over %d examples (disabled, or not enough text to fit "
+ "one); the model will judge on the domain name alone", len(examples))
self.text = None
blocks.append(self._shape_block(examples))
@@ -128,8 +152,9 @@ class Featuriser:
@staticmethod
def _shape_block(examples: list[ModerationExample]) -> csr_matrix:
- shapes = np.array([shape_features(example.domain) for example in examples], dtype=float)
- return csr_matrix(shapes / SHAPE_SCALE)
+ counts = np.array([shape_features(example.domain) for example in examples], dtype=float)
+ indicators = np.array([indicator_features(example) for example in examples], dtype=float)
+ return csr_matrix(np.hstack([counts / SHAPE_SCALE, indicators]))
def feature_names(self) -> np.ndarray:
"""Feature labels in matrix order, for turning coefficients into moderator-facing text."""
@@ -140,7 +165,5 @@ class Featuriser:
if self.text is not None:
names.append(np.array([f"text={name}"
for name in self.text.get_feature_names_out()]))
- names.append(np.array([
- "len", "labels", "digits", "hyphens", "www", "sld_len", "spamwords", "tld_len",
- ]))
+ names.append(np.array(COUNT_FEATURE_NAMES + INDICATOR_FEATURE_NAMES))
return np.concatenate(names)
diff --git a/mwmbl/moderation/model.py b/mwmbl/moderation/model.py
index beaacd5..9f3f8e2 100644
--- a/mwmbl/moderation/model.py
+++ b/mwmbl/moderation/model.py
@@ -91,11 +91,22 @@ class Suggestion:
class ModerationModel:
"""A fitted featuriser plus the two heads, pickled together as one artifact."""
- def __init__(self, featuriser: Featuriser, reject_head, reason_head, version: str):
+ # Class-level default so an artifact pickled before training domains were recorded still
+ # answers the question. Instances always set their own in __init__.
+ train_domains: set = frozenset()
+
+ def __init__(self, featuriser: Featuriser, reject_head, reason_head, version: str,
+ train_domains: Optional[set] = None):
self.featuriser = featuriser
self.reject_head = reject_head
self.reason_head = reason_head
self.version = version
+ # What this model was fitted on, so a later retrain can exclude those rows when it
+ # scores this model on its own held-out set. Without it, the incumbent is measured
+ # partly on data it memorised and the comparison quietly favours whatever is deployed.
+ # Empty for artifacts pickled before this was recorded; mwmbl.moderation.train reports
+ # that rather than assuming either way.
+ self.train_domains = train_domains or set()
def predict(self, examples: list[ModerationExample]) -> list[tuple[float, str, float]]:
"""(reject probability, reason, reason probability) for each example, in one pass."""
@@ -192,6 +203,17 @@ def get_model() -> Optional[ModerationModel]:
return _model
+def load_published_model() -> Optional[ModerationModel]:
+ """The artifact the workers are serving, for the retrain to compare itself against.
+
+ Deliberately not :func:`get_model`, whose cache exists to keep the request path cheap: a
+ retrain wants the row as it stands right now, and it wants it without disturbing what the
+ rest of the process is serving.
+ """
+ published = _published_stamp()
+ return _load(published[0] if published else None)
+
+
def publish(model: ModerationModel, metrics: dict) -> ModerationModelArtifact:
"""Store a trained model and its metrics as the artifact every worker will serve."""
buffer = io.BytesIO()
@@ -229,6 +251,37 @@ def _published_stamp() -> Optional[tuple[str, datetime]]:
return (row["version"], row["created_on"]) if row else None
+def is_compatible(model: ModerationModel) -> bool:
+ """Whether a loaded artifact can actually score a domain with the code that loaded it.
+
+ A featuriser is pickled whole, so a change to the feature set - one more shape column, a
+ block that moved - leaves old artifacts describing a matrix this code no longer builds, and
+ the heads raise on the width mismatch at *predict* time. In the enrichment task that means
+ every domain fails to be scored rather than one, and the failure looks like a crawl problem
+ rather than what it is.
+
+ So loading ends with one probe prediction, and a model that cannot answer it is treated
+ exactly like a model that could not be unpickled: dropped, with the deterministic checks
+ carrying on alone. A probe rather than a version stamp because it catches whatever drifted,
+ not only the drift somebody remembered to bump a number for.
+ """
+ try:
+ model.predict([ModerationExample("example.com", [])])
+ except Exception:
+ logger.exception("Moderation model %s could not score a probe domain", model.version)
+ return False
+ return True
+
+
+def _checked(model: Optional[ModerationModel], where: str) -> Optional[ModerationModel]:
+ if model is None or is_compatible(model):
+ return model
+ logger.error("Moderation model %s from %s was fitted on a different feature set than this "
+ "code builds; ignoring it and falling back to the deterministic checks. "
+ "Retrain to publish a compatible artifact.", model.version, where)
+ return None
+
+
def _load(version: Optional[str]) -> Optional[ModerationModel]:
if version is None:
return _load_bundled()
@@ -243,7 +296,7 @@ def _load(version: Optional[str]) -> Optional[ModerationModel]:
"artifact", version)
return _load_bundled()
logger.info("Domain moderation model %s loaded from the database", model.version)
- return model
+ return _checked(model, "the database")
def _load_bundled() -> Optional[ModerationModel]:
@@ -260,7 +313,7 @@ def _load_bundled() -> Optional[ModerationModel]:
logger.exception("Failed to load the domain moderation model from %s", model_path)
return None
logger.info("Domain moderation model %s loaded from %s", model.version, model_path)
- return model
+ return _checked(model, str(model_path))
def load_metrics() -> dict:
diff --git a/mwmbl/moderation/train.py b/mwmbl/moderation/train.py
index 9082060..e537b59 100644
--- a/mwmbl/moderation/train.py
+++ b/mwmbl/moderation/train.py
@@ -1,16 +1,28 @@
"""Fit and evaluate the moderation suggester.
-The evaluation is the point of this module, not the fitting. Two things about this data make
+The evaluation is the point of this module, not the fitting. Three things about this data make
a naive accuracy number actively misleading:
-* One prolific submitter accounts for 799 of the last 949 decisions at a 1% rejection rate,
- so an aggregate score is dominated by rows no model is needed for. Cold-start submitters -
- the 16% where a moderator is genuinely deciding something - reject at 54%, and that slice is
- what the gate reads.
-* That slice has ~81 positives, where point estimates move around far more than they look
- like they do. Bootstrapping the augmentation experiment turned a 0.04 PR-AUC "improvement"
- into two intervals spanning 0.17 that overlapped almost entirely. So the gate compares
- intervals, and every metric is reported with one.
+* One prolific submitter accounts for most decisions at a ~1% rejection rate, so an aggregate
+ score is dominated by rows no model is needed for. Cold-start submitters - the minority where
+ a moderator is genuinely deciding something - reject far more often, and that slice is what
+ the gate reads.
+* That slice has a couple of hundred positives, where point estimates move around far more than
+ they look like they do. Bootstrapping the augmentation experiment turned a 0.04 PR-AUC
+ "improvement" into two intervals spanning 0.17 that overlapped almost entirely. So every
+ metric is reported with an interval.
+* **PR-AUC's floor is the positive rate, so it is not comparable across populations.** The
+ August 2026 retrain looked like a 0.11 regression (0.779 -> 0.672 cold-start PR-AUC) purely
+ because migration 0037 repaired 1,785 previously-invisible decisions into the training set
+ and moved the cold-start slice's rejection rate from 54% to 30%. On the prevalence-corrected
+ measure the "regression" was a small *improvement* (0.519 -> 0.532). Every slice therefore
+ reports ``base_rate`` and ``normalised_ap`` - ``(AP - base) / (1 - base)``, which is 0 for a
+ coin flip and 1 for a perfect ranker regardless of prevalence - and the gate reads that.
+
+The gate's real answer, though, is not a number compared against a stored number from a run on
+a different population. It is :func:`compare_models`: score the incumbent and the candidate on
+the *same* held-out rows and bootstrap the *difference*, which cancels the shared test-set
+variance and is far more sensitive than comparing two independent intervals.
"""
from __future__ import annotations
@@ -33,27 +45,52 @@ TEST_FRACTION = 0.25
BOOTSTRAP_SAMPLES = 400
MIN_CLASS_ROWS = 5
+# Non-inferiority margin, in normalised-AP points, for the paired comparison against the
+# incumbent. Requiring the difference to be significantly *positive* would ship nothing: with a
+# couple of hundred positives, a genuine improvement of a few points does not clear zero. So the
+# gate asks the answerable question instead - are we confident this is not meaningfully worse -
+# and 0.05 is about a third of the width of a typical interval on this data.
+GATE_TOLERANCE = 0.05
+
@dataclass
class Evaluation:
slice_name: str
rows: int
positives: int
+ base_rate: float
pr_auc: float
pr_auc_ci: tuple[float, float]
+ normalised_ap: float
+ normalised_ap_ci: tuple[float, float]
recall_at_precision_75: float
+ rows_with_text: int
def to_dict(self) -> dict:
return {
"slice": self.slice_name,
"rows": self.rows,
"positives": self.positives,
+ # Reported next to every score because PR-AUC cannot be read without it: the same
+ # model scores 0.78 on a 54%-positive slice and 0.67 on a 30%-positive one.
+ "base_rate": round(self.base_rate, 4),
"pr_auc": round(self.pr_auc, 4),
"pr_auc_ci": [round(self.pr_auc_ci[0], 4), round(self.pr_auc_ci[1], 4)],
+ "normalised_ap": round(self.normalised_ap, 4),
+ "normalised_ap_ci": [round(self.normalised_ap_ci[0], 4),
+ round(self.normalised_ap_ci[1], 4)],
"recall_at_precision_75": round(self.recall_at_precision_75, 4),
+ # Evidence is crawled newest-first, so this differs sharply between the splits and
+ # is the number to look at before believing anything about the text features.
+ "rows_with_text": self.rows_with_text,
}
+def normalise(average_precision: float, base_rate: float) -> float:
+ """PR-AUC rescaled so chance is 0 and perfect is 1, making it comparable across slices."""
+ return (average_precision - base_rate) / (1 - base_rate)
+
+
def split_by_time(rows: list[TrainingRow]) -> tuple[list[TrainingRow], list[TrainingRow]]:
"""Time-ordered split of the real rows; derived and seed rows always go to training.
@@ -68,13 +105,20 @@ def split_by_time(rows: list[TrainingRow]) -> tuple[list[TrainingRow], list[Trai
return real[:cut] + other, real[cut:]
-def train(rows: list[TrainingRow], version: str) -> tuple[ModerationModel, dict]:
- """Fit both heads and evaluate on held-out real rows only."""
+def train(rows: list[TrainingRow], version: str, use_text: bool = True,
+ incumbent: Optional[ModerationModel] = None) -> tuple[ModerationModel, dict]:
+ """Fit both heads and evaluate on held-out real rows only.
+
+ ``use_text=False`` fits the domain-name-only model, which is the ablation that answers
+ whether the crawled page text is earning its place. ``incumbent`` is the model currently
+ being served; when given, the metrics carry a paired comparison against it on these exact
+ test rows, which is the only comparison that means anything.
+ """
train_rows, test_rows = split_by_time(rows)
logger.info("Training on %d rows (%d real), testing on %d real rows",
len(train_rows), sum(r.source == REAL for r in train_rows), len(test_rows))
- featuriser = Featuriser()
+ featuriser = Featuriser(use_text=use_text)
features = featuriser.fit_transform([row.to_example() for row in train_rows])
# The binary head sees real rows only: blanket augmentation was measured and did not help.
@@ -89,64 +133,155 @@ def train(rows: list[TrainingRow], version: str) -> tuple[ModerationModel, dict]
reason_head.fit(features[reason_mask],
[row.reason for row, keep in zip(train_rows, reason_mask) if keep])
- model = ModerationModel(featuriser, reject_head, reason_head, version)
- return model, evaluate(model, train_rows, test_rows)
+ model = ModerationModel(featuriser, reject_head, reason_head, version,
+ train_domains={row.domain for row in train_rows})
+ return model, evaluate(model, train_rows, test_rows, incumbent)
def evaluate(model: ModerationModel, train_rows: list[TrainingRow],
- test_rows: list[TrainingRow]) -> dict:
+ test_rows: list[TrainingRow],
+ incumbent: Optional[ModerationModel] = None) -> dict:
if not test_rows:
return {"error": "no held-out real rows to evaluate on"}
- predictions = model.predict([row.to_example() for row in test_rows])
+ examples = [row.to_example() for row in test_rows]
+ predictions = model.predict(examples)
reject_scores = np.array([prediction[0] for prediction in predictions])
truth = np.array([row.rejected for row in test_rows], dtype=int)
+ has_text = np.array([bool(row.to_example().text.strip()) for row in test_rows])
# The slice that matters. One prolific submitter accounts for most decisions at a 1%
# rejection rate, so an aggregate score mostly measures rows no model is needed for.
- # Submissions from an account with no track record in the training window reject at 54%,
- # and those are the ones a moderator is genuinely deciding.
+ # Submissions from an account with no track record in the training window reject far more
+ # often, and those are the ones a moderator is genuinely deciding.
known_submitters = {row.submitter for row in train_rows if row.submitter}
cold = np.array([row.submitter not in known_submitters for row in test_rows])
slices = {"all": np.ones(len(test_rows), dtype=bool), "cold_start": cold}
metrics = {
- name: _evaluate_slice(name, truth[mask], reject_scores[mask]).to_dict()
+ name: _evaluate_slice(name, truth[mask], reject_scores[mask], has_text[mask]).to_dict()
for name, mask in slices.items()
if _has_both_classes(truth[mask])
}
metrics["reason_head"] = _evaluate_reason_head(predictions, test_rows)
metrics["train_rows_by_source"] = _count_sources(train_rows)
+ # Fit-time text coverage, alongside the per-slice test coverage above. A large gap between
+ # the two is the train/test skew that makes the text block look worse than it is.
+ metrics["train_rows_with_text"] = sum(
+ 1 for row in train_rows if row.to_example().text.strip())
+ metrics["uses_text"] = model.featuriser.text is not None
metrics["suggestion_influence"] = _suggestion_influence(train_rows + test_rows)
+ if incumbent is not None:
+ metrics["versus_incumbent"] = compare_models(
+ model, incumbent, test_rows, cold, reject_scores)
return metrics
-def _evaluate_slice(name: str, truth: np.ndarray, scores: np.ndarray) -> Evaluation:
+def compare_models(candidate: ModerationModel, incumbent: ModerationModel,
+ test_rows: list[TrainingRow], cold: np.ndarray,
+ candidate_scores: np.ndarray) -> dict:
+ """Paired bootstrap of candidate minus incumbent on identical cold-start rows.
+
+ Two models scored on the same rows share all the variance that comes from *which* rows
+ landed in the test set, and resampling them together cancels it. Comparing two independent
+ intervals - what the gate used to do, across two different test sets - throws that away and
+ then blames the model for the difference.
+
+ Rows whose domain the incumbent was trained on are dropped. Scoring a model on its own
+ training data flatters it, and since the incumbent is the thing being defended, leaving them
+ in biases the comparison *against* the candidate. Older artifacts carry no record of what
+ they trained on; the comparison still runs, and says so, because a candidate that wins under
+ a bias in the incumbent's favour has won.
+ """
+ incumbent_scores = np.array([prediction[0] for prediction in
+ incumbent.predict([row.to_example() for row in test_rows])])
+
+ contamination_known = bool(incumbent.train_domains)
+ seen = incumbent.train_domains or set()
+ clean = np.array([row.domain not in seen for row in test_rows])
+ mask = cold & clean
+ if not _has_both_classes(np.array([row.rejected for row in test_rows], dtype=int)[mask]):
+ return {"note": "not enough uncontaminated cold-start rows to compare on",
+ "incumbent_version": incumbent.version}
+
+ truth = np.array([row.rejected for row in test_rows], dtype=int)[mask]
+ candidate_slice = candidate_scores[mask]
+ incumbent_slice = incumbent_scores[mask]
+
+ generator = np.random.default_rng(0)
+ differences = []
+ for _ in range(BOOTSTRAP_SAMPLES):
+ indices = generator.choice(len(truth), len(truth), replace=True)
+ if not _has_both_classes(truth[indices]):
+ continue
+ base = truth[indices].mean()
+ differences.append(
+ normalise(average_precision_score(truth[indices], candidate_slice[indices]), base)
+ - normalise(average_precision_score(truth[indices], incumbent_slice[indices]), base))
+
+ base_rate = float(truth.mean())
+ difference = (normalise(average_precision_score(truth, candidate_slice), base_rate)
+ - normalise(average_precision_score(truth, incumbent_slice), base_rate))
+ return {
+ "slice": "cold_start",
+ "incumbent_version": incumbent.version,
+ "rows": int(mask.sum()),
+ "positives": int(truth.sum()),
+ "rows_dropped_as_incumbent_training_data": int((cold & ~clean).sum()),
+ # False means the incumbent predates train-domain recording, so some of these rows may
+ # be its own training data and the comparison leans in its favour.
+ "contamination_known": contamination_known,
+ "candidate_normalised_ap": round(
+ normalise(average_precision_score(truth, candidate_slice), base_rate), 4),
+ "incumbent_normalised_ap": round(
+ normalise(average_precision_score(truth, incumbent_slice), base_rate), 4),
+ "difference": round(difference, 4),
+ "difference_ci": [round(float(np.percentile(differences, 2.5)), 4),
+ round(float(np.percentile(differences, 97.5)), 4)],
+ # How often the candidate won a resample. Reads more directly than the interval when
+ # the interval straddles zero, which on this much data it usually will.
+ "candidate_wins_fraction": round(float(np.mean(np.array(differences) > 0)), 4),
+ }
+
+
+def _evaluate_slice(name: str, truth: np.ndarray, scores: np.ndarray,
+ has_text: np.ndarray) -> Evaluation:
generator = np.random.default_rng(0)
- samples = []
+ samples, normalised = [], []
for _ in range(BOOTSTRAP_SAMPLES):
indices = generator.choice(len(truth), len(truth), replace=True)
- if _has_both_classes(truth[indices]):
- samples.append(average_precision_score(truth[indices], scores[indices]))
+ if not _has_both_classes(truth[indices]):
+ continue
+ average_precision = average_precision_score(truth[indices], scores[indices])
+ samples.append(average_precision)
+ # Normalised per resample, against that resample's own base rate: the resampling itself
+ # moves prevalence around, and correcting with the whole-slice rate would leave that in.
+ normalised.append(normalise(average_precision, truth[indices].mean()))
precision, recall, _ = precision_recall_curve(truth, scores)
+ base_rate = float(truth.mean())
return Evaluation(
slice_name=name,
rows=len(truth),
positives=int(truth.sum()),
+ base_rate=base_rate,
pr_auc=float(average_precision_score(truth, scores)),
pr_auc_ci=(float(np.percentile(samples, 2.5)) if samples else 0.0,
float(np.percentile(samples, 97.5)) if samples else 0.0),
+ normalised_ap=normalise(float(average_precision_score(truth, scores)), base_rate),
+ normalised_ap_ci=(float(np.percentile(normalised, 2.5)) if normalised else 0.0,
+ float(np.percentile(normalised, 97.5)) if normalised else 0.0),
recall_at_precision_75=max(
(r for p, r in zip(precision, recall) if p >= 0.75 and r > 0), default=0.0),
+ rows_with_text=int(has_text.sum()),
)
def _evaluate_reason_head(predictions, test_rows: list[TrainingRow]) -> dict:
"""Per-reason F1 on the real rejections we held out.
- Per class rather than averaged, because the classes are not comparable: SPAM has 208 real
- examples behind it and OFFENSIVE has one. An average would let a collapse in the class
+ Per class rather than averaged, because the classes are not comparable: SPAM has hundreds of
+ real examples behind it and OFFENSIVE has one. An average would let a collapse in the class
that works be hidden by the class that never did.
"""
truth, predicted = [], []
@@ -206,24 +341,73 @@ def _count_sources(rows: list[TrainingRow]) -> dict:
return dict(Counter(row.source for row in rows))
+def normalised_ap_of(slice_metrics: dict) -> Optional[tuple[float, float]]:
+ """Point estimate and interval lower bound in normalised AP, from either metrics format.
+
+ Artifacts stored before normalised AP existed recorded ``pr_auc`` and enough to reconstruct
+ the base rate, so a retrain landing on top of one is not forced back to comparing raw PR-AUC
+ across populations - the mistake this whole module exists to stop. Read through this on both
+ sides of the comparison rather than only the incumbent's, so a rollback to older code, or a
+ hand-written metrics blob, is a weaker comparison rather than a KeyError in the gate.
+ """
+ if "normalised_ap" in slice_metrics:
+ return slice_metrics["normalised_ap"], slice_metrics["normalised_ap_ci"][0]
+ if not slice_metrics.get("rows"):
+ return None
+ base_rate = slice_metrics["positives"] / slice_metrics["rows"]
+ return (normalise(slice_metrics["pr_auc"], base_rate),
+ normalise(slice_metrics["pr_auc_ci"][0], base_rate))
+
+
def passes_gate(new: dict, current: dict) -> tuple[bool, str]:
"""Whether a freshly trained model may replace the deployed one.
- The comparison is on the cold-start slice, and against the *lower* bound of the incumbent's
- interval rather than its point estimate. A gate on point estimates would have shipped the
- augmentation change that bootstrapping later showed to be noise.
+ Preference order, best evidence first:
+
+ 1. The paired comparison on identical rows, when the incumbent model could be loaded. The
+ candidate ships unless it is *confidently* worse by more than :data:`GATE_TOLERANCE`.
+ 2. Otherwise, normalised AP against the lower bound of the incumbent's stored interval.
+ Weaker, because the two numbers come from different test sets - but at least it is not
+ comparing raw PR-AUC across different prevalences, which failed the August 2026 retrain
+ for a regression that had not happened.
"""
new_cold = new.get("cold_start")
if new_cold is None:
return False, "no cold-start slice in the new metrics - not enough held-out data"
+ candidate = normalised_ap_of(new_cold)
+ if candidate is None:
+ return False, "the new cold-start metrics cannot be read"
+
+ comparison = new.get("versus_incumbent")
+ if comparison is not None and "difference" in comparison:
+ caveat = "" if comparison["contamination_known"] else (
+ "; the incumbent has no record of its training domains, so some of these rows may "
+ "be its own and the comparison favours it")
+ if comparison["difference_ci"][0] < -GATE_TOLERANCE:
+ return False, (
+ f"paired against {comparison['incumbent_version']} on {comparison['rows']} "
+ f"cold-start rows, normalised AP is {comparison['difference']:+.3f} with "
+ f"interval [{comparison['difference_ci'][0]:.3f}, "
+ f"{comparison['difference_ci'][1]:.3f}], which admits a loss worse than the "
+ f"{GATE_TOLERANCE:.2f} tolerance{caveat}")
+ return True, (
+ f"paired against {comparison['incumbent_version']} on {comparison['rows']} "
+ f"cold-start rows, normalised AP is {comparison['difference']:+.3f} "
+ f"[{comparison['difference_ci'][0]:.3f}, {comparison['difference_ci'][1]:.3f}], "
+ f"winning {comparison['candidate_wins_fraction']:.0%} of resamples{caveat}")
+
current_cold = current.get("cold_start")
- if current_cold is None:
- return True, f"no incumbent to compare against; cold-start PR-AUC {new_cold['pr_auc']:.3f}"
-
- floor = current_cold["pr_auc_ci"][0]
- if new_cold["pr_auc"] < floor:
- return False, (f"cold-start PR-AUC {new_cold['pr_auc']:.3f} is below the incumbent's "
- f"lower bound {floor:.3f}")
- return True, (f"cold-start PR-AUC {new_cold['pr_auc']:.3f} "
- f"(incumbent {current_cold['pr_auc']:.3f}, lower bound {floor:.3f})")
+ incumbent = normalised_ap_of(current_cold) if current_cold else None
+ if incumbent is None:
+ return True, (f"no readable incumbent to compare against; cold-start normalised AP "
+ f"{candidate[0]:.3f}")
+
+ point, floor = incumbent
+ if candidate[0] < floor:
+ return False, (
+ f"cold-start normalised AP {candidate[0]:.3f} is below the incumbent's lower bound "
+ f"{floor:.3f} (unpaired: the incumbent model could not be loaded, so these are "
+ f"different test sets and the comparison is weak)")
+ return True, (f"cold-start normalised AP {candidate[0]:.3f} "
+ f"(incumbent {point:.3f}, lower bound {floor:.3f}; unpaired)")
diff --git a/scripts/moderation_eval.py b/scripts/moderation_eval.py
index 031479a..93244b1 100644
--- a/scripts/moderation_eval.py
+++ b/scripts/moderation_eval.py
@@ -6,6 +6,7 @@ sanitized judgments export, so the evaluation is reproducible on a laptop and in
uv run python scripts/moderation_eval.py
uv run python scripts/moderation_eval.py --derived 2000 # needs network
+ uv run python scripts/moderation_eval.py --write-artifact # regenerate the warm start
Reports every metric with a bootstrap interval. That is not decoration: the cold-start slice
has ~80 positives, and the augmentation experiment that motivated the derived-data design
@@ -16,8 +17,11 @@ import gzip
import json
import os
import sys
+from datetime import date
from pathlib import Path
+import joblib
+
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
import django
@@ -25,10 +29,12 @@ import django
django.setup()
from mwmbl.moderation.train import train # noqa: E402
+from mwmbl.moderation.model import METRICS_FILENAME, MODEL_FILENAME # noqa: E402
from mwmbl.moderation.training_data import ( # noqa: E402
REAL, TrainingRow, is_trainable_domain, derived_rows, seed_rows)
DEFAULT_EXPORT = Path("devdata/judgments_export/domains.jsonl.gz")
+ARTIFACT_DIR = Path("mwmbl/moderation/artifacts")
def rows_from_export(path: Path) -> list[TrainingRow]:
@@ -61,6 +67,12 @@ def main() -> int:
parser.add_argument("--derived", type=int, default=0,
help="Blocklist rows per under-represented reason (0 = skip the fetch)")
parser.add_argument("--no-seed", action="store_true")
+ parser.add_argument(
+ "--write-artifact", action="store_true",
+ help=(f"Overwrite the committed warm start in {ARTIFACT_DIR}. Needed after a change to "
+ "the feature set: a featuriser is pickled whole, so the shipped artifact then "
+ "describes a matrix the code no longer builds, fails the load-time probe, and "
+ "every deploy degrades to the deterministic checks until the first retrain."))
options = parser.parse_args()
if not options.export.exists():
@@ -77,8 +89,18 @@ def main() -> int:
rows += derived_rows(["OFFENSIVE"], options.derived,
exclude={row.domain for row in rows})
- _, metrics = train(rows, "offline-eval")
+ # Dated like a retrain's, because this artifact is served in production until the first
+ # retrain publishes and "offline-eval" in a worker log says nothing about how old it is.
+ version = (f"domain-mod-{date.today():%Y-%m-%d}" if options.write_artifact
+ else "offline-eval")
+ model, metrics = train(rows, version)
print(json.dumps(metrics, indent=2))
+
+ if options.write_artifact:
+ joblib.dump(model, ARTIFACT_DIR / MODEL_FILENAME)
+ (ARTIFACT_DIR / METRICS_FILENAME).write_text(json.dumps(metrics, indent=2) + "\n")
+ print(f"\nWrote the warm start to {ARTIFACT_DIR}. This is the day-one model only - a "
+ f"retrain against the production database still replaces it.")
return 0
diff --git a/scripts/moderation_llm_bakeoff.py b/scripts/moderation_llm_bakeoff.py
new file mode 100644
index 0000000..92a597d
--- /dev/null
+++ b/scripts/moderation_llm_bakeoff.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Compare an LLM's approve/reject judgements against the trained suggester, on identical rows.
+
+The question this answers is not "how accurate is the LLM" - accuracy on a slice whose base
+rate moves is not a number you can compare to anything. It is "does the LLM rank the domains a
+moderator will reject above the ones they will approve, better than the model we ship?", scored
+with exactly the machinery in mwmbl.moderation.train so the two numbers are on one scale.
+
+Three steps, because the middle one runs outside this process:
+
+ # 1. write the held-out cold-start rows, domain names only, with no labels in the file
+ uv run python scripts/moderation_llm_bakeoff.py export --out devdata/moderation_bakeoff
+
+ # 2. have an LLM judge devdata/moderation_bakeoff/domains.txt into a JSONL of
+ # {"domain": ..., "reject_probability": 0-100} and save it beside that file
+
+ # 3. score every judgement file against the model, paired, on the same rows
+ uv run python scripts/moderation_llm_bakeoff.py score --dir devdata/moderation_bakeoff
+
+The export deliberately carries no labels and no submitter, so the file handed to a judge -
+which may be a third party - cannot leak the answer or say who submitted what.
+"""
+import argparse
+import gzip
+import json
+import os
+import sys
+from pathlib import Path
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
+
+import django
+
+django.setup()
+
+import numpy as np # noqa: E402
+from sklearn.metrics import average_precision_score # noqa: E402
+
+from mwmbl.moderation.train import ( # noqa: E402
+ BOOTSTRAP_SAMPLES, normalise, split_by_time, train)
+from mwmbl.moderation.training_data import seed_rows # noqa: E402
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from moderation_eval import DEFAULT_EXPORT, rows_from_export # noqa: E402
+
+DOMAINS_FILE = "domains.txt"
+TRUTH_FILE = "truth.jsonl"
+JUDGEMENT_SUFFIX = ".judgements.jsonl"
+
+
+def cold_start_test_rows(export: Path):
+ """The held-out rows from submitters with no track record, and the model's scores for them.
+
+ Rebuilt here rather than passed in, so the rows an LLM is asked about are the same rows the
+ gate reads - a bake-off on a different slice would answer a different question.
+ """
+ rows = rows_from_export(export) + seed_rows()
+ model, _ = train(rows, "bakeoff")
+ train_rows, test_rows = split_by_time(rows)
+
+ known = {row.submitter for row in train_rows if row.submitter}
+ cold = [row for row in test_rows if row.submitter not in known]
+ scores = [prediction[0] for prediction in
+ model.predict([row.to_example() for row in cold])]
+ return cold, np.array(scores)
+
+
+def export(options) -> int:
+ rows, scores = cold_start_test_rows(options.export)
+ options.out.mkdir(parents=True, exist_ok=True)
+
+ (options.out / DOMAINS_FILE).write_text(
+ "".join(f"{row.domain}\n" for row in rows))
+ # The labels and the model's own scores stay on this side, keyed by domain, so scoring can
+ # join them back without the judge ever having seen them.
+ with (options.out / TRUTH_FILE).open("w") as truth:
+ for row, score in zip(rows, scores):
+ truth.write(json.dumps({"domain": row.domain, "rejected": row.rejected,
+ "reason": row.reason, "model_score": float(score)}) + "\n")
+
+ print(f"{len(rows)} cold-start held-out domains -> {options.out / DOMAINS_FILE}")
+ print(f"{sum(row.rejected for row in rows)} of them were rejected "
+ f"(base rate {sum(row.rejected for row in rows) / len(rows):.3f})")
+ return 0
+
+
+def read_judgements(path: Path) -> dict:
+ """A judge's file as {domain: reject probability in [0, 1]}.
+
+ Tolerant about the scale because judges are asked for 0-100 and some answer 0-1, but the
+ scale is decided **once for the whole file**, not per value. Deciding per value reads a
+ judge's single most confident approval - a 1 out of 100 - as a certainty of 1.0, turning it
+ into the most confident *rejection* in the file. Both judges in the first bake-off emitted
+ exactly one such row, and it cost each of them real score.
+
+ Not tolerant about anything else: a malformed line is a judgement we cannot score, and
+ silently dropping it would quietly change the slice.
+ """
+ records = []
+ for line in path.read_text().splitlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ record = json.loads(line)
+ records.append((record["domain"], float(record["reject_probability"])))
+
+ percentages = any(score > 1 for _, score in records)
+ return {domain: score / 100 if percentages else score for domain, score in records}
+
+
+def paired_bootstrap(truth: np.ndarray, left: np.ndarray, right: np.ndarray) -> dict:
+ """Normalised AP for two rankers over the same rows, and an interval on the difference.
+
+ Resampling both on the same indices cancels the variance that comes from which rows landed
+ in the test set, which is most of it on a slice this size.
+ """
+ generator = np.random.default_rng(0)
+ differences = []
+ for _ in range(BOOTSTRAP_SAMPLES):
+ indices = generator.choice(len(truth), len(truth), replace=True)
+ sample = truth[indices]
+ if sample.sum() < 5 or (1 - sample).sum() < 5:
+ continue
+ base = sample.mean()
+ differences.append(normalise(average_precision_score(sample, left[indices]), base)
+ - normalise(average_precision_score(sample, right[indices]), base))
+
+ base_rate = float(truth.mean())
+ return {
+ "left": round(normalise(average_precision_score(truth, left), base_rate), 4),
+ "right": round(normalise(average_precision_score(truth, right), base_rate), 4),
+ "difference": round(
+ normalise(average_precision_score(truth, left), base_rate)
+ - normalise(average_precision_score(truth, right), base_rate), 4),
+ "difference_ci": [round(float(np.percentile(differences, 2.5)), 4),
+ round(float(np.percentile(differences, 97.5)), 4)],
+ "left_wins_fraction": round(float(np.mean(np.array(differences) > 0)), 4),
+ }
+
+
+def score(options) -> int:
+ truth_rows = [json.loads(line) for line in
+ (options.dir / TRUTH_FILE).read_text().splitlines() if line.strip()]
+ # One entry per distinct domain, not per submission. A domain submitted nine times is one
+ # thing a moderator decides, and counting it nine times would weight the score by how
+ # popular a submission was rather than by how hard it was to judge. Both rankers are
+ # deduplicated the same way, so the comparison is unaffected either way.
+ by_domain = {row["domain"]: row for row in truth_rows}
+ print(f"{len(truth_rows)} held-out submissions, {len(by_domain)} distinct domains")
+
+ files = sorted(options.dir.glob(f"*{JUDGEMENT_SUFFIX}"))
+ if not files:
+ print(f"No *{JUDGEMENT_SUFFIX} files in {options.dir}", file=sys.stderr)
+ return 1
+
+ for path in files:
+ judgements = read_judgements(path)
+ # Only rows this judge actually answered, so a judge that skipped the hard ones is
+ # visible as coverage rather than hidden in a score computed over a smaller slice.
+ answered = [domain for domain in by_domain if domain in judgements]
+ truth = np.array([by_domain[domain]["rejected"] for domain in answered], dtype=int)
+ llm = np.array([judgements[domain] for domain in answered])
+ model = np.array([by_domain[domain]["model_score"] for domain in answered])
+
+ result = paired_bootstrap(truth, llm, model)
+ print(f"\n{path.name}")
+ print(f" answered {len(answered)}/{len(by_domain)} domains, "
+ f"{int(truth.sum())} rejected (base rate {truth.mean():.3f})")
+ print(f" normalised AP: LLM {result['left']:.4f} model {result['right']:.4f} "
+ f"difference {result['difference']:+.4f} "
+ f"[{result['difference_ci'][0]:+.4f}, {result['difference_ci'][1]:+.4f}]")
+ print(f" the LLM ranks better in {result['left_wins_fraction']:.0%} of resamples")
+ print(f" raw PR-AUC: LLM "
+ f"{average_precision_score(truth, llm):.4f} model "
+ f"{average_precision_score(truth, model):.4f}")
+ return 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ subparsers = parser.add_subparsers(dest="command", required=True)
+
+ exporter = subparsers.add_parser("export")
+ exporter.add_argument("--export", type=Path, default=DEFAULT_EXPORT)
+ exporter.add_argument("--out", type=Path, default=Path("devdata/moderation_bakeoff"))
+ exporter.set_defaults(function=export)
+
+ scorer = subparsers.add_parser("score")
+ scorer.add_argument("--dir", type=Path, default=Path("devdata/moderation_bakeoff"))
+ scorer.set_defaults(function=score)
+
+ options = parser.parse_args()
+ return options.function(options)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/test/test_domain_moderation.py b/test/test_domain_moderation.py
index 78eedcf..3147464 100644
--- a/test/test_domain_moderation.py
+++ b/test/test_domain_moderation.py
@@ -7,7 +7,9 @@ queue never runs a model on the request path.
"""
import io
import json
+import sys
from datetime import timedelta
+from pathlib import Path
from unittest import mock
import joblib
@@ -180,6 +182,69 @@ def test_featuriser_produces_the_same_width_for_unseen_input():
assert len(featuriser.feature_names()) == fitted.shape[1]
+def test_has_text_separates_an_uncrawled_domain_from_one_whose_words_are_unknown():
+ """Both produce an all-zero TF-IDF row, and they mean opposite things. Evidence is crawled
+ newest-first, so under a chronological split most training rows have no text and most test
+ rows do; without this indicator the model cannot tell the two cases apart and the
+ difference is absorbed into an intercept fitted on the training mix."""
+ featuriser = Featuriser()
+ featuriser.fit_transform([
+ ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),
+ ModerationExample("docs.python.org", ["python language reference tool"]),
+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),
+ ])
+ assert featuriser.text is not None
+ column = list(featuriser.feature_names()).index("has_text")
+
+ uncrawled = featuriser.transform([ModerationExample("a.example", [])])
+ unmatched = featuriser.transform([ModerationExample("a.example", ["kanji zzz"])])
+ assert uncrawled[0, column] == 0.0
+ assert unmatched[0, column] == 1.0
+
+
+def test_the_ablation_fits_the_same_features_minus_the_text_block():
+ """--no-text has to run through the real featuriser, not a copy of it, or the ablation
+ measures the copy."""
+ examples = [ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),
+ ModerationExample("docs.python.org", ["python language reference tool"]),
+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"])]
+ with_text = Featuriser().fit_transform(examples).shape[1]
+ without = Featuriser(use_text=False)
+ assert without.fit_transform(examples).shape[1] < with_text
+ assert without.text is None
+ assert "has_text" in list(without.feature_names())
+
+
+def test_a_judges_most_confident_approval_is_not_read_as_a_certain_rejection(tmp_path):
+ """scripts/moderation_llm_bakeoff accepts judgements on either a 0-1 or a 0-100 scale, and
+ the scale has to be decided per file. Deciding per value reads a 1 out of 100 - the most
+ confident approval a judge can express - as a probability of 1.0, the most confident
+ rejection in the file. Both judges in the first bake-off emitted exactly one such row."""
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+ from moderation_llm_bakeoff import read_judgements
+
+ path = tmp_path / "judge.judgements.jsonl"
+ path.write_text(
+ '{"domain": "obviously-fine.org", "reject_probability": 1, "reason": ""}\n'
+ '{"domain": "aislopgenerator.xyz", "reject_probability": 95, "reason": "SPAM"}\n')
+
+ judgements = read_judgements(path)
+ assert judgements["obviously-fine.org"] == 0.01
+ assert judgements["aislopgenerator.xyz"] == 0.95
+
+
+def test_judgements_already_on_a_zero_to_one_scale_are_left_alone(tmp_path):
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+ from moderation_llm_bakeoff import read_judgements
+
+ path = tmp_path / "judge.judgements.jsonl"
+ path.write_text(
+ '{"domain": "obviously-fine.org", "reject_probability": 0.01, "reason": ""}\n'
+ '{"domain": "aislopgenerator.xyz", "reject_probability": 0.95, "reason": "SPAM"}\n')
+
+ assert read_judgements(path) == {"obviously-fine.org": 0.01, "aislopgenerator.xyz": 0.95}
+
+
def test_malformed_historic_names_are_excluded_from_training():
"""Migrations 0032/0033 fixed these at the API layer; training on them teaches a
problem that can no longer occur."""
@@ -789,8 +854,22 @@ def test_redirect_check_errs_towards_reporting_nothing():
# --------------------------------------------------------------------- retrain gate
-def metrics(pr_auc, low, high):
- return {"cold_start": {"pr_auc": pr_auc, "pr_auc_ci": [low, high]}}
+def metrics(pr_auc, low, high, rows=150, positives=81):
+ """Cold-start metrics in the shape a stored artifact carries.
+
+ rows and positives are what make the PR-AUC readable: the gate corrects for prevalence, so
+ a slice cannot be described by its score alone.
+ """
+ return {"cold_start": {"pr_auc": pr_auc, "pr_auc_ci": [low, high],
+ "rows": rows, "positives": positives}}
+
+
+def paired(difference, low, high, contamination_known=True):
+ return {"cold_start": metrics(0.7, 0.6, 0.8)["cold_start"],
+ "versus_incumbent": {
+ "incumbent_version": "domain-mod-2026-08-01", "rows": 150,
+ "contamination_known": contamination_known, "difference": difference,
+ "difference_ci": [low, high], "candidate_wins_fraction": 0.5}}
def test_gate_publishes_when_there_is_no_incumbent():
@@ -802,7 +881,7 @@ def test_gate_blocks_a_model_below_the_incumbents_lower_bound():
allowed, explanation = passes_gate(metrics(0.60, 0.51, 0.69),
metrics(0.78, 0.69, 0.86))
assert not allowed
- assert "0.690" in explanation
+ assert "unpaired" in explanation
def test_gate_tolerates_movement_inside_the_interval():
@@ -813,6 +892,53 @@ def test_gate_tolerates_movement_inside_the_interval():
assert allowed
+def test_gate_is_not_fooled_by_a_change_in_prevalence():
+ """The August 2026 retrain, exactly as it happened.
+
+ Migration 0037 repaired 1,785 previously-invisible decisions into the training set, which
+ moved the cold-start rejection rate from 54% to 30%. PR-AUC's floor *is* the positive rate,
+ so the same quality of ranking scored 0.672 instead of 0.779 and the gate called it a
+ regression. Corrected for prevalence it is a small improvement, 0.519 -> 0.532, and it
+ ships."""
+ allowed, explanation = passes_gate(
+ metrics(0.6716, 0.5927, 0.7472, rows=635, positives=189),
+ metrics(0.7788, 0.6885, 0.8588, rows=150, positives=81))
+ assert allowed, explanation
+
+
+def test_the_paired_comparison_is_preferred_to_the_stored_metrics():
+ """Two numbers from two different test sets can only be compared weakly. When the incumbent
+ could be loaded and scored on the same rows, that is the answer - even when the stored
+ metrics, describing a different population, would have said otherwise."""
+ allowed, explanation = passes_gate(paired(0.02, -0.03, 0.07),
+ metrics(0.95, 0.93, 0.97))
+ assert allowed
+ assert "paired against domain-mod-2026-08-01" in explanation
+
+
+def test_gate_blocks_a_candidate_that_is_confidently_worse():
+ allowed, explanation = passes_gate(paired(-0.18, -0.26, -0.09), metrics(0.5, 0.4, 0.6))
+ assert not allowed
+ assert "tolerance" in explanation
+
+
+def test_gate_ships_a_small_loss_it_cannot_distinguish_from_noise():
+ """Non-inferiority, not superiority. Requiring a significant *gain* on a slice with ~80
+ positives would never ship anything, including the models that are genuinely better."""
+ allowed, _ = passes_gate(paired(-0.02, -0.045, 0.01), metrics(0.5, 0.4, 0.6))
+ assert allowed
+
+
+def test_an_incumbent_with_no_training_record_is_flagged_not_trusted_silently():
+ """Artifacts pickled before train_domains was recorded may be scored on rows they were
+ fitted on, which flatters them. The candidate still has to clear the bar - the point is
+ that the explanation says the bar was tilted."""
+ allowed, explanation = passes_gate(paired(0.01, -0.02, 0.05, contamination_known=False),
+ metrics(0.5, 0.4, 0.6))
+ assert allowed
+ assert "no record of its training domains" in explanation
+
+
def test_gate_refuses_when_there_is_no_cold_start_slice():
"""Too little held-out data to judge is a reason not to publish, not a free pass."""
allowed, explanation = passes_gate({"all": {"pr_auc": 0.9}}, {})
@@ -1135,6 +1261,14 @@ class _StubModel:
return [(self.score, "SPAM", self.score) for _ in examples]
+class _StaleFeatureSetModel:
+ """An artifact whose pickled featuriser no longer matches the code that loaded it."""
+ version = "stale"
+
+ def predict(self, examples):
+ raise ValueError("X has 8 features, but LogisticRegression is expecting 9 features")
+
+
def artifact_bytes(model) -> bytes:
buffer = io.BytesIO()
joblib.dump(model, buffer)
@@ -1210,6 +1344,20 @@ def test_an_unloadable_artifact_falls_back_rather_than_failing(tmp_path, setting
assert get_model().version == "bundled"
+def test_an_artifact_from_a_different_feature_set_degrades_to_the_rules(tmp_path, settings):
+ """A featuriser is pickled whole, so adding a feature leaves every stored artifact building
+ a matrix its heads were not fitted on. That raises at predict time, inside the enrichment
+ task, for every domain - so loading probes the model once and drops it if it cannot answer,
+ which is the same degradation an unloadable pickle already gets."""
+ joblib.dump(_StaleFeatureSetModel(), tmp_path / "model.joblib")
+ settings.DOMAIN_MODERATION_MODEL_DIR = str(tmp_path)
+
+ assert get_model() is None
+ suggestion = suggest("a.com", [], [])
+ assert (suggestion.action, suggestion.model_version) == ("UNSURE", "")
+
+
# --------------------------------------------------------------------- request validation
@pytest.mark.django_db