An open source, non-profit web search engine
Include domain submissions in the judgments export
Daoud Clarked9763d84c314
diff --git a/mwmbl/management/commands/export_judgments.py b/mwmbl/management/commands/export_judgments.pyindex 3547492..21dad81 100644--- a/mwmbl/management/commands/export_judgments.py+++ b/mwmbl/management/commands/export_judgments.py@@ -9,6 +9,8 @@ Dumps three gzipped JSONL files plus a stats summary to an output directory:- ``user_curations.jsonl.gz`` one record per UserCuration action event (oldinterface): move/delete/add/validate with the result list at that point.- ``votes.jsonl.gz`` one record per SearchResultVote (query, url, ±).+- ``domains.jsonl.gz`` one record per DomainSubmission: a human label at+ domain granularity (REJECTED/SPAM = junk domain, APPROVED = acceptable).- ``stats.json`` counts and distributions, also printed, so a``--stats-only`` run works as a data health check before exporting.@@ -26,7 +28,7 @@ from urllib.parse import parse_qs, urlparsefrom django.core.management.base import BaseCommandfrom django.utils.dateparse import parse_date-from mwmbl.models import Curation, FlagCuration, SearchResultVote, UserCuration+from mwmbl.models import Curation, DomainSubmission, FlagCuration, SearchResultVote, UserCurationDOCUMENT_FIELDS = ("url", "title", "extract", "score", "state")@@ -77,13 +79,14 @@ class Command(BaseCommand):writers = {}if not stats_only:output_dir.mkdir(parents=True, exist_ok=True)- for name in ("curations", "user_curations", "votes"):+ for name in ("curations", "user_curations", "votes", "domains"):writers[name] = gzip.open(output_dir / f"{name}.jsonl.gz", "wt")try:stats["curations"] = self.export_curations(writers.get("curations"), anon, since)stats["user_curations"] = self.export_user_curations(writers.get("user_curations"), anon, since)stats["votes"] = self.export_votes(writers.get("votes"), anon, since)+ stats["domains"] = self.export_domains(writers.get("domains"), anon, since)finally:for writer in writers.values():writer.close()@@ -237,6 +240,45 @@ class Command(BaseCommand):}+ def export_domains(self, writer, anon, since):+ queryset = DomainSubmission.objects.order_by("id")+ if since:+ queryset = queryset.filter(submitted_on__date__gte=since)++ count = 0+ status_counts = Counter()+ rejection_reason_counts = Counter()+ domains = set()+ timestamps = MinMax()++ for submission in queryset.iterator():+ record = {+ "domain": submission.name,+ "user": anon.get(submission.submitted_by_id),+ "timestamp": submission.submitted_on.isoformat() if submission.submitted_on else None,+ "status": submission.status,+ "rejection_reason": submission.rejection_reason or None,+ "rejection_detail": submission.rejection_detail or None,+ }+ if writer:+ writer.write(json.dumps(record) + "\n")++ count += 1+ status_counts[submission.status] += 1+ if submission.rejection_reason:+ rejection_reason_counts[submission.rejection_reason] += 1+ domains.add(submission.name)+ timestamps.add(submission.submitted_on)++ return {+ "count": count,+ "distinct_domains": len(domains),+ "status_counts": dict(status_counts),+ "rejection_reason_counts": dict(rejection_reason_counts),+ "timestamp_range": timestamps.range(),+ }++class AnonymousUserIds:"""Stable opaque per-export user ids, shared across all exported files."""diff --git a/test/test_export_judgments.py b/test/test_export_judgments.pyindex 399096b..10ab132 100644--- a/test/test_export_judgments.py+++ b/test/test_export_judgments.py@@ -6,7 +6,7 @@ import pytestfrom django.contrib.auth import get_user_modelfrom django.core.management import call_command-from mwmbl.models import Curation, FlagCuration, SearchResultVote, UserCuration+from mwmbl.models import Curation, DomainSubmission, FlagCuration, SearchResultVote, UserCurationUser = get_user_model()@@ -53,11 +53,14 @@ def test_export_writes_sanitized_records(tmp_path, user, flagger, curation):)SearchResultVote.objects.create(user=user, url="https://a.com", query="test query", vote_type="upvote")+ DomainSubmission.objects.create(+ name="spammy.example.com", submitted_by=user, status="REJECTED",+ status_changed_by=flagger, rejection_reason="SPAM", rejection_detail="link farm")call_command("export_judgments", output_dir=str(tmp_path))records = {}- for name in ("curations", "user_curations", "votes"):+ for name in ("curations", "user_curations", "votes", "domains"):with gzip.open(tmp_path / f"{name}.jsonl.gz", "rt") as f:records[name] = [json.loads(line) for line in f]@@ -82,11 +85,21 @@ def test_export_writes_sanitized_records(tmp_path, user, flagger, curation):assert vote["vote_type"] == "upvote"assert vote["user"] == exported["user"]+ [domain] = records["domains"]+ assert domain["domain"] == "spammy.example.com"+ assert domain["status"] == "REJECTED"+ assert domain["rejection_reason"] == "SPAM"+ assert domain["user"] == exported["user"]+ # the reviewing moderator's identity is not exported+ assert "flagger" not in json.dumps(records)+stats = json.loads((tmp_path / "stats.json").read_text())assert stats["curations"]["count"] == 1assert stats["curations"]["flag_status_counts"] == {"ACCEPTED": 1}assert stats["user_curations"]["curation_type_counts"] == {"curate_delete": 1}assert stats["votes"]["vote_type_counts"] == {"upvote": 1}+ assert stats["domains"]["status_counts"] == {"REJECTED": 1}+ assert stats["domains"]["rejection_reason_counts"] == {"SPAM": 1}assert stats["users"]["distinct_users"] == 1