Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Keep the stored text's author last in user_ids

Claudeb79269d36d3a
diff --git a/mwmbl/crawler/agreement.py b/mwmbl/crawler/agreement.py
index 1930480..2ddb871 100644
--- a/mwmbl/crawler/agreement.py
+++ b/mwmbl/crawler/agreement.py
@@ -15,7 +15,7 @@ def record_agreements(agreements: set[Agreement], user: MwmblUser, crawler_versi
CrawlerAgreement(
url=agreement.url,
user=user,
- existing_user_ids=list(agreement.existing_user_ids),
+ existing_user_id=agreement.existing_user_id,
title_similarity=agreement.title_similarity,
extract_similarity=agreement.extract_similarity,
last_crawled_delta=agreement.last_crawled_delta,
diff --git a/mwmbl/indexer/agreement.py b/mwmbl/indexer/agreement.py
index fcb17c8..126adc2 100644
--- a/mwmbl/indexer/agreement.py
+++ b/mwmbl/indexer/agreement.py
@@ -14,9 +14,9 @@ from mwmbl.tinysearchengine.indexer import Document
class Agreement(NamedTuple):
url: str
- existing_user_ids: tuple[int, ...]
- """The crawlers the stored copy is attributed to. Up to two of them, and the text is
- whichever of their versions ranked higher, so it cannot be pinned on just one."""
+ existing_user_id: int
+ """The crawler whose text the stored copy holds: the last of its user_ids, which
+ combine_documents keeps for the kept copy's author."""
title_similarity: float
extract_similarity: float
last_crawled_delta: Optional[int]
@@ -28,21 +28,21 @@ class Agreement(NamedTuple):
def find_agreements(incoming: Iterable[Document], existing: Iterable[Document]) -> set[Agreement]:
"""Compare each incoming document against another crawler's stored copy of its URL."""
# A page keeps one document per URL, so there is at most one stored copy to compare.
- attributed_by_url = {document.url: document for document in existing if document.user_ids}
+ authors_by_url = {document.url: (document, document.user_ids[-1]) for document in existing if document.user_ids}
agreements = set()
for document in incoming:
- stored = attributed_by_url.get(document.url)
- if stored is None or document.user_ids is None:
+ if document.url not in authors_by_url or document.user_ids is None:
continue
- # The stored copy may be this crawler's own, from an earlier submission.
- if set(document.user_ids) & set(stored.user_ids):
+ stored, existing_user_id = authors_by_url[document.url]
+ # The stored text may be this crawler's own, from an earlier submission.
+ if existing_user_id in document.user_ids:
continue
both_crawled = document.last_crawled is not None and stored.last_crawled is not None
last_crawled_delta = document.last_crawled - stored.last_crawled if both_crawled else None
agreements.add(
Agreement(
url=document.url,
- existing_user_ids=tuple(stored.user_ids),
+ existing_user_id=existing_user_id,
title_similarity=trigram_similarity(document.title, stored.title),
extract_similarity=trigram_similarity(document.extract, stored.extract),
last_crawled_delta=last_crawled_delta,
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index bef5e22..2640d5e 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -31,6 +31,8 @@ MAX_USER_IDS = 2
def _merge_user_ids(existing: Optional[list[int]], incoming: Optional[list[int]]) -> Optional[list[int]]:
+ """The incoming ids go last, moved there if already present, and the oldest are dropped
+ past MAX_USER_IDS."""
combined = list(existing or [])
for uid in incoming or []:
if uid in combined:
@@ -301,7 +303,10 @@ def combine_documents(existing_documents, documents, mark_synced, ranker):
continue
if mark_synced:
document.state = DocumentState.SYNCED_WITH_MAIN_INDEX
- document.user_ids = url_user_ids.get(document.url)
+ # The kept copy's ids merged last, so the last id is always the crawler whose text the
+ # page holds - see mwmbl.indexer.agreement. Its own last id is its author by the same
+ # rule, and a fresh submission carries only its submitter's.
+ document.user_ids = _merge_user_ids(url_user_ids.get(document.url), document.user_ids)
document.last_crawled = url_last_crawled.get(document.url)
combined_documents.append(document)
seen_urls.add(document.url)
diff --git a/mwmbl/migrations/0048_crawleragreement.py b/mwmbl/migrations/0048_crawleragreement.py
index 73090d4..bc7b8ab 100644
--- a/mwmbl/migrations/0048_crawleragreement.py
+++ b/mwmbl/migrations/0048_crawleragreement.py
@@ -1,6 +1,5 @@
-# Generated by Django 5.2.13 on 2026-10-17 21:50
+# Generated by Django 5.2.13 on 2026-10-18 08:03
-import django.contrib.postgres.fields
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
@@ -25,10 +24,7 @@ class Migration(migrations.Migration):
),
),
("url", models.TextField()),
- (
- "existing_user_ids",
- django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None),
- ),
+ ("existing_user_id", models.IntegerField(db_index=True)),
("title_similarity", models.FloatField()),
("extract_similarity", models.FloatField()),
("last_crawled_delta", models.IntegerField(null=True)),
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 87f561e..6575eaa 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -591,8 +591,8 @@ class CrawlerAgreement(models.Model):
url = models.TextField()
user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE, related_name="+")
- # Not foreign keys: up to two users, and the stored text is one of theirs without saying which.
- existing_user_ids = ArrayField(models.IntegerField())
+ # Not a foreign key: it comes from the index, which keeps the ids of deleted users.
+ existing_user_id = models.IntegerField(db_index=True)
title_similarity = models.FloatField()
extract_similarity = models.FloatField()
last_crawled_delta = models.IntegerField(null=True) # seconds, incoming crawl minus stored crawl
diff --git a/test/test_crawler_agreement.py b/test/test_crawler_agreement.py
index 88e4c8f..7b3de6c 100644
--- a/test/test_crawler_agreement.py
+++ b/test/test_crawler_agreement.py
@@ -43,7 +43,7 @@ def test_find_agreements_compares_against_another_crawlers_copy():
(agreement,) = find_agreements(incoming, existing)
assert agreement.url == URL
- assert agreement.existing_user_ids == (1, 2)
+ assert agreement.existing_user_id == 2
assert agreement.title_similarity == 1.0
assert agreement.extract_similarity == 0.0
assert agreement.last_crawled_delta == 600
@@ -56,6 +56,16 @@ def test_find_agreements_skips_the_crawlers_own_copy():
assert find_agreements(incoming, existing) == set()
+def test_find_agreements_compares_against_another_crawlers_text_the_crawler_is_credited_on():
+ """The crawler is on the stored copy's user_ids, but the text is the last one's."""
+ existing = [Document("Title", URL, "Old extract", term="t", user_ids=[3, 1])]
+ incoming = [Document("Title", URL, "New extract", term="t", user_ids=[3])]
+
+ (agreement,) = find_agreements(incoming, existing)
+
+ assert agreement.existing_user_id == 1
+
+
def test_find_agreements_skips_unattributed_documents():
"""Seed crawls and the main index sync write documents with no user_ids: there is no
crawler to hold them against."""
@@ -109,7 +119,7 @@ def test_index_documents_reports_one_agreement_per_url(index_path, no_blacklist)
assert indexed.agreements == {
Agreement(
url=URL,
- existing_user_ids=(1,),
+ existing_user_id=1,
title_similarity=1.0,
extract_similarity=1.0,
last_crawled_delta=None,
@@ -134,7 +144,7 @@ def test_index_documents_reports_the_agreement_whichever_copy_wins(index_path, n
indexed = index_documents([Document(title, URL, extract, user_ids=[2])], index_path)
(agreement,) = indexed.agreements
- assert agreement.existing_user_ids == (1,)
+ assert agreement.existing_user_id == 1
assert agreement.title_similarity == trigram_similarity(title, "Python tutorial")
assert agreement.extract_similarity == trigram_similarity(extract, "Learn python today")
@@ -168,7 +178,7 @@ def user(db):
def _agreement(url=URL):
return Agreement(
url=url,
- existing_user_ids=(1, 2),
+ existing_user_id=2,
title_similarity=0.9,
extract_similarity=0.4,
last_crawled_delta=60,
@@ -182,7 +192,7 @@ def test_record_agreements(user):
stored = CrawlerAgreement.objects.get()
assert stored.url == URL
assert stored.user == user
- assert stored.existing_user_ids == [1, 2]
+ assert stored.existing_user_id == 2
assert stored.title_similarity == 0.9
assert stored.extract_similarity == 0.4
assert stored.last_crawled_delta == 60
@@ -225,7 +235,7 @@ def test_post_results_records_agreements(user, index_path, no_blacklist):
assert response.status_code == 200
stored = CrawlerAgreement.objects.get()
assert stored.user == user
- assert stored.existing_user_ids == [other_user.id]
+ assert stored.existing_user_id == other_user.id
assert stored.title_similarity == 1.0
assert stored.extract_similarity < 1.0
assert stored.crawler_version == "1.2.0"
diff --git a/test/test_index_batches.py b/test/test_index_batches.py
index 8a4784e..9508a05 100644
--- a/test/test_index_batches.py
+++ b/test/test_index_batches.py
@@ -212,6 +212,32 @@ def test_combine_documents_propagates_user_ids_to_winner():
assert 2 in combined[0].user_ids
+ "existing_ids, existing_extract, incoming_extract, expected",
+ [
+ ([1], "a longer extract", "short", [3, 1]),
+ ([1], "short", "a longer extract", [1, 3]),
+ # The kept copy's ids keep their order, its author last, when the cap drops the rest.
+ ([1, 2], "a longer extract", "short", [1, 2]),
+ ],
+)
+def test_combine_documents_puts_the_kept_texts_author_last(existing_ids, existing_extract, incoming_extract, expected):
+ """mwmbl.indexer.agreement reads the author of a stored copy's text off the end of its
+ user_ids, whichever copy ranks first."""
+ existing = [Document(title="t", url="http://a.com", extract=existing_extract, term="q", user_ids=existing_ids)]
+ new_docs = [Document(title="t", url="http://a.com", extract=incoming_extract, term="q", user_ids=[3])]
+
+ (kept,) = combine_documents(existing, new_docs, mark_synced=False, ranker=LongestExtractRanker())
+
+ assert kept.user_ids == expected
+
+
+class LongestExtractRanker:
+ @staticmethod
+ def order_results(terms: list[str], pages: list[Document], is_complete: bool):
+ return sorted(pages, key=lambda doc: -len(doc.extract))
+
+
# ---------------------------------------------------------------------------
# index_documents: blacklist enforcement
# ---------------------------------------------------------------------------