An open source, non-profit web search engine
Keep the stored text's author last in user_ids
Claudeb79269d36d3a
diff --git a/mwmbl/crawler/agreement.py b/mwmbl/crawler/agreement.pyindex 1930480..2ddb871 100644--- a/mwmbl/crawler/agreement.py+++ b/mwmbl/crawler/agreement.py@@ -15,7 +15,7 @@ def record_agreements(agreements: set[Agreement], user: MwmblUser, crawler_versiCrawlerAgreement(url=agreement.url,user=user,- existing_user_ids=list(agreement.existing_user_ids),+ existing_user_id=agreement.existing_user_id,title_similarity=agreement.title_similarity,extract_similarity=agreement.extract_similarity,last_crawled_delta=agreement.last_crawled_delta,diff --git a/mwmbl/indexer/agreement.py b/mwmbl/indexer/agreement.pyindex fcb17c8..126adc2 100644--- a/mwmbl/indexer/agreement.py+++ b/mwmbl/indexer/agreement.py@@ -14,9 +14,9 @@ from mwmbl.tinysearchengine.indexer import Documentclass Agreement(NamedTuple):url: str- existing_user_ids: tuple[int, ...]- """The crawlers the stored copy is attributed to. Up to two of them, and the text is- whichever of their versions ranked higher, so it cannot be pinned on just one."""+ existing_user_id: int+ """The crawler whose text the stored copy holds: the last of its user_ids, which+ combine_documents keeps for the kept copy's author."""title_similarity: floatextract_similarity: floatlast_crawled_delta: Optional[int]@@ -28,21 +28,21 @@ class Agreement(NamedTuple):def find_agreements(incoming: Iterable[Document], existing: Iterable[Document]) -> set[Agreement]:"""Compare each incoming document against another crawler's stored copy of its URL."""# A page keeps one document per URL, so there is at most one stored copy to compare.- attributed_by_url = {document.url: document for document in existing if document.user_ids}+ authors_by_url = {document.url: (document, document.user_ids[-1]) for document in existing if document.user_ids}agreements = set()for document in incoming:- stored = attributed_by_url.get(document.url)- if stored is None or document.user_ids is None:+ if document.url not in authors_by_url or document.user_ids is None:continue- # The stored copy may be this crawler's own, from an earlier submission.- if set(document.user_ids) & set(stored.user_ids):+ stored, existing_user_id = authors_by_url[document.url]+ # The stored text may be this crawler's own, from an earlier submission.+ if existing_user_id in document.user_ids:continueboth_crawled = document.last_crawled is not None and stored.last_crawled is not Nonelast_crawled_delta = document.last_crawled - stored.last_crawled if both_crawled else Noneagreements.add(Agreement(url=document.url,- existing_user_ids=tuple(stored.user_ids),+ existing_user_id=existing_user_id,title_similarity=trigram_similarity(document.title, stored.title),extract_similarity=trigram_similarity(document.extract, stored.extract),last_crawled_delta=last_crawled_delta,diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex bef5e22..2640d5e 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -31,6 +31,8 @@ MAX_USER_IDS = 2def _merge_user_ids(existing: Optional[list[int]], incoming: Optional[list[int]]) -> Optional[list[int]]:+ """The incoming ids go last, moved there if already present, and the oldest are dropped+ past MAX_USER_IDS."""combined = list(existing or [])for uid in incoming or []:if uid in combined:@@ -301,7 +303,10 @@ def combine_documents(existing_documents, documents, mark_synced, ranker):continueif mark_synced:document.state = DocumentState.SYNCED_WITH_MAIN_INDEX- document.user_ids = url_user_ids.get(document.url)+ # The kept copy's ids merged last, so the last id is always the crawler whose text the+ # page holds - see mwmbl.indexer.agreement. Its own last id is its author by the same+ # rule, and a fresh submission carries only its submitter's.+ document.user_ids = _merge_user_ids(url_user_ids.get(document.url), document.user_ids)document.last_crawled = url_last_crawled.get(document.url)combined_documents.append(document)seen_urls.add(document.url)diff --git a/mwmbl/migrations/0048_crawleragreement.py b/mwmbl/migrations/0048_crawleragreement.pyindex 73090d4..bc7b8ab 100644--- a/mwmbl/migrations/0048_crawleragreement.py+++ b/mwmbl/migrations/0048_crawleragreement.py@@ -1,6 +1,5 @@-# Generated by Django 5.2.13 on 2026-10-17 21:50+# Generated by Django 5.2.13 on 2026-10-18 08:03-import django.contrib.postgres.fieldsimport django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models@@ -25,10 +24,7 @@ class Migration(migrations.Migration):),),("url", models.TextField()),- (- "existing_user_ids",- django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None),- ),+ ("existing_user_id", models.IntegerField(db_index=True)),("title_similarity", models.FloatField()),("extract_similarity", models.FloatField()),("last_crawled_delta", models.IntegerField(null=True)),diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 87f561e..6575eaa 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -591,8 +591,8 @@ class CrawlerAgreement(models.Model):url = models.TextField()user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE, related_name="+")- # Not foreign keys: up to two users, and the stored text is one of theirs without saying which.- existing_user_ids = ArrayField(models.IntegerField())+ # Not a foreign key: it comes from the index, which keeps the ids of deleted users.+ existing_user_id = models.IntegerField(db_index=True)title_similarity = models.FloatField()extract_similarity = models.FloatField()last_crawled_delta = models.IntegerField(null=True) # seconds, incoming crawl minus stored crawldiff --git a/test/test_crawler_agreement.py b/test/test_crawler_agreement.pyindex 88e4c8f..7b3de6c 100644--- a/test/test_crawler_agreement.py+++ b/test/test_crawler_agreement.py@@ -43,7 +43,7 @@ def test_find_agreements_compares_against_another_crawlers_copy():(agreement,) = find_agreements(incoming, existing)assert agreement.url == URL- assert agreement.existing_user_ids == (1, 2)+ assert agreement.existing_user_id == 2assert agreement.title_similarity == 1.0assert agreement.extract_similarity == 0.0assert agreement.last_crawled_delta == 600@@ -56,6 +56,16 @@ def test_find_agreements_skips_the_crawlers_own_copy():assert find_agreements(incoming, existing) == set()+def test_find_agreements_compares_against_another_crawlers_text_the_crawler_is_credited_on():+ """The crawler is on the stored copy's user_ids, but the text is the last one's."""+ existing = [Document("Title", URL, "Old extract", term="t", user_ids=[3, 1])]+ incoming = [Document("Title", URL, "New extract", term="t", user_ids=[3])]++ (agreement,) = find_agreements(incoming, existing)++ assert agreement.existing_user_id == 1++def test_find_agreements_skips_unattributed_documents():"""Seed crawls and the main index sync write documents with no user_ids: there is nocrawler to hold them against."""@@ -109,7 +119,7 @@ def test_index_documents_reports_one_agreement_per_url(index_path, no_blacklist)assert indexed.agreements == {Agreement(url=URL,- existing_user_ids=(1,),+ existing_user_id=1,title_similarity=1.0,extract_similarity=1.0,last_crawled_delta=None,@@ -134,7 +144,7 @@ def test_index_documents_reports_the_agreement_whichever_copy_wins(index_path, nindexed = index_documents([Document(title, URL, extract, user_ids=[2])], index_path)(agreement,) = indexed.agreements- assert agreement.existing_user_ids == (1,)+ assert agreement.existing_user_id == 1assert agreement.title_similarity == trigram_similarity(title, "Python tutorial")assert agreement.extract_similarity == trigram_similarity(extract, "Learn python today")@@ -168,7 +178,7 @@ def user(db):def _agreement(url=URL):return Agreement(url=url,- existing_user_ids=(1, 2),+ existing_user_id=2,title_similarity=0.9,extract_similarity=0.4,last_crawled_delta=60,@@ -182,7 +192,7 @@ def test_record_agreements(user):stored = CrawlerAgreement.objects.get()assert stored.url == URLassert stored.user == user- assert stored.existing_user_ids == [1, 2]+ assert stored.existing_user_id == 2assert stored.title_similarity == 0.9assert stored.extract_similarity == 0.4assert stored.last_crawled_delta == 60@@ -225,7 +235,7 @@ def test_post_results_records_agreements(user, index_path, no_blacklist):assert response.status_code == 200stored = CrawlerAgreement.objects.get()assert stored.user == user- assert stored.existing_user_ids == [other_user.id]+ assert stored.existing_user_id == other_user.idassert stored.title_similarity == 1.0assert stored.extract_similarity < 1.0assert stored.crawler_version == "1.2.0"diff --git a/test/test_index_batches.py b/test/test_index_batches.pyindex 8a4784e..9508a05 100644--- a/test/test_index_batches.py+++ b/test/test_index_batches.py@@ -212,6 +212,32 @@ def test_combine_documents_propagates_user_ids_to_winner():assert 2 in combined[0].user_ids+ "existing_ids, existing_extract, incoming_extract, expected",+ [+ ([1], "a longer extract", "short", [3, 1]),+ ([1], "short", "a longer extract", [1, 3]),+ # The kept copy's ids keep their order, its author last, when the cap drops the rest.+ ([1, 2], "a longer extract", "short", [1, 2]),+ ],+)+def test_combine_documents_puts_the_kept_texts_author_last(existing_ids, existing_extract, incoming_extract, expected):+ """mwmbl.indexer.agreement reads the author of a stored copy's text off the end of its+ user_ids, whichever copy ranks first."""+ existing = [Document(title="t", url="http://a.com", extract=existing_extract, term="q", user_ids=existing_ids)]+ new_docs = [Document(title="t", url="http://a.com", extract=incoming_extract, term="q", user_ids=[3])]++ (kept,) = combine_documents(existing, new_docs, mark_synced=False, ranker=LongestExtractRanker())++ assert kept.user_ids == expected+++class LongestExtractRanker:+ @staticmethod+ def order_results(terms: list[str], pages: list[Document], is_complete: bool):+ return sorted(pages, key=lambda doc: -len(doc.extract))++# ---------------------------------------------------------------------------# index_documents: blacklist enforcement# ---------------------------------------------------------------------------