Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Include domain submissions in the judgments export

Daoud Clarked9763d84c314
diff --git a/mwmbl/management/commands/export_judgments.py b/mwmbl/management/commands/export_judgments.py
index 3547492..21dad81 100644
--- a/mwmbl/management/commands/export_judgments.py
+++ b/mwmbl/management/commands/export_judgments.py
@@ -9,6 +9,8 @@ Dumps three gzipped JSONL files plus a stats summary to an output directory:
- ``user_curations.jsonl.gz`` one record per UserCuration action event (old
interface): move/delete/add/validate with the result list at that point.
- ``votes.jsonl.gz`` one record per SearchResultVote (query, url, ±).
+- ``domains.jsonl.gz`` one record per DomainSubmission: a human label at
+ domain granularity (REJECTED/SPAM = junk domain, APPROVED = acceptable).
- ``stats.json`` counts and distributions, also printed, so a
``--stats-only`` run works as a data health check before exporting.
@@ -26,7 +28,7 @@ from urllib.parse import parse_qs, urlparse
from django.core.management.base import BaseCommand
from django.utils.dateparse import parse_date
-from mwmbl.models import Curation, FlagCuration, SearchResultVote, UserCuration
+from mwmbl.models import Curation, DomainSubmission, FlagCuration, SearchResultVote, UserCuration
DOCUMENT_FIELDS = ("url", "title", "extract", "score", "state")
@@ -77,13 +79,14 @@ class Command(BaseCommand):
writers = {}
if not stats_only:
output_dir.mkdir(parents=True, exist_ok=True)
- for name in ("curations", "user_curations", "votes"):
+ for name in ("curations", "user_curations", "votes", "domains"):
writers[name] = gzip.open(output_dir / f"{name}.jsonl.gz", "wt")
try:
stats["curations"] = self.export_curations(writers.get("curations"), anon, since)
stats["user_curations"] = self.export_user_curations(writers.get("user_curations"), anon, since)
stats["votes"] = self.export_votes(writers.get("votes"), anon, since)
+ stats["domains"] = self.export_domains(writers.get("domains"), anon, since)
finally:
for writer in writers.values():
writer.close()
@@ -237,6 +240,45 @@ class Command(BaseCommand):
}
+ def export_domains(self, writer, anon, since):
+ queryset = DomainSubmission.objects.order_by("id")
+ if since:
+ queryset = queryset.filter(submitted_on__date__gte=since)
+
+ count = 0
+ status_counts = Counter()
+ rejection_reason_counts = Counter()
+ domains = set()
+ timestamps = MinMax()
+
+ for submission in queryset.iterator():
+ record = {
+ "domain": submission.name,
+ "user": anon.get(submission.submitted_by_id),
+ "timestamp": submission.submitted_on.isoformat() if submission.submitted_on else None,
+ "status": submission.status,
+ "rejection_reason": submission.rejection_reason or None,
+ "rejection_detail": submission.rejection_detail or None,
+ }
+ if writer:
+ writer.write(json.dumps(record) + "\n")
+
+ count += 1
+ status_counts[submission.status] += 1
+ if submission.rejection_reason:
+ rejection_reason_counts[submission.rejection_reason] += 1
+ domains.add(submission.name)
+ timestamps.add(submission.submitted_on)
+
+ return {
+ "count": count,
+ "distinct_domains": len(domains),
+ "status_counts": dict(status_counts),
+ "rejection_reason_counts": dict(rejection_reason_counts),
+ "timestamp_range": timestamps.range(),
+ }
+
+
class AnonymousUserIds:
"""Stable opaque per-export user ids, shared across all exported files."""
diff --git a/test/test_export_judgments.py b/test/test_export_judgments.py
index 399096b..10ab132 100644
--- a/test/test_export_judgments.py
+++ b/test/test_export_judgments.py
@@ -6,7 +6,7 @@ import pytest
from django.contrib.auth import get_user_model
from django.core.management import call_command
-from mwmbl.models import Curation, FlagCuration, SearchResultVote, UserCuration
+from mwmbl.models import Curation, DomainSubmission, FlagCuration, SearchResultVote, UserCuration
User = get_user_model()
@@ -53,11 +53,14 @@ def test_export_writes_sanitized_records(tmp_path, user, flagger, curation):
)
SearchResultVote.objects.create(
user=user, url="https://a.com", query="test query", vote_type="upvote")
+ DomainSubmission.objects.create(
+ name="spammy.example.com", submitted_by=user, status="REJECTED",
+ status_changed_by=flagger, rejection_reason="SPAM", rejection_detail="link farm")
call_command("export_judgments", output_dir=str(tmp_path))
records = {}
- for name in ("curations", "user_curations", "votes"):
+ for name in ("curations", "user_curations", "votes", "domains"):
with gzip.open(tmp_path / f"{name}.jsonl.gz", "rt") as f:
records[name] = [json.loads(line) for line in f]
@@ -82,11 +85,21 @@ def test_export_writes_sanitized_records(tmp_path, user, flagger, curation):
assert vote["vote_type"] == "upvote"
assert vote["user"] == exported["user"]
+ [domain] = records["domains"]
+ assert domain["domain"] == "spammy.example.com"
+ assert domain["status"] == "REJECTED"
+ assert domain["rejection_reason"] == "SPAM"
+ assert domain["user"] == exported["user"]
+ # the reviewing moderator's identity is not exported
+ assert "flagger" not in json.dumps(records)
+
stats = json.loads((tmp_path / "stats.json").read_text())
assert stats["curations"]["count"] == 1
assert stats["curations"]["flag_status_counts"] == {"ACCEPTED": 1}
assert stats["user_curations"]["curation_type_counts"] == {"curate_delete": 1}
assert stats["votes"]["vote_type_counts"] == {"upvote": 1}
+ assert stats["domains"]["status_counts"] == {"REJECTED": 1}
+ assert stats["domains"]["rejection_reason_counts"] == {"SPAM": 1}
assert stats["users"]["distinct_users"] == 1