An open source, non-profit web search engine
Super Search: track per-URL source provenance + propagate to crawls
Daoud Clarkec293a7812c9a
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex bdd2125..9181cec 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -50,6 +50,7 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Reurl_timestamps = {}url_statuses = defaultdict(lambda: URLStatus.NEW)domain_links = defaultdict(set)+ crawled_page_links: dict[str, list[str]] = {}for batch in batches:for item in batch.items:timestamp = get_datetime_from_timestamp(item.timestamp / 1000.0)@@ -66,7 +67,7 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Recontinuefor link in item.content.all_links:process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,- blacklist_provider, domain_links)+ blacklist_provider, domain_links, item.url, crawled_page_links)found_urls = [FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])for url in url_statuses.keys() | url_users.keys()]@@ -79,6 +80,8 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Refor source_domain, target_domains in domain_links.items():domain_link_db.update_domain_links(source_domain, target_domains)+ _propagate_source_provenance(crawled_page_links)+end = datetime.utcnow()logger.info(f"Recorded URLs in database in {end - start}")@@ -108,7 +111,7 @@ def add_hn_links(new_item_queue: RedisURLQueue):def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timestamp: datetime,url_timestamps: dict[str, datetime], url_users: dict[str, str], blacklist_provider,- domain_links: dict[str, set[str]]):+ domain_links: dict[str, set[str]], page_url: str, crawled_page_links: dict[str, list[str]]):try:parsed_link = parse_url(link.url)except ValueError:@@ -129,6 +132,52 @@ def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timesturl_users[root_url] = user_id_hashurl_timestamps[root_url] = timestampdomain_links[crawled_page_domain].add(parsed_link.netloc)+ # Remember which links came off which crawled page, for source-provenance+ # propagation (only used when the page itself has a known Super Search source).+ if len(link.url) <= 500:+ crawled_page_links.setdefault(page_url, []).append(link.url)+++def _propagate_source_provenance(crawled_page_links: dict[str, list[str]]):+ """Carry a crawled page's Super Search source onto the links found on it.++ Only pages already recorded in SourceProvenance propagate, so the common case+ (no provenance) costs a single batched query. Propagation is bounded by+ SOURCE_PROVENANCE_MAX_DEPTH to limit transitive spread. No-op without a+ database; never raises into URL recording.+ """+ if not getattr(settings, "HAS_DATABASE", False):+ return+ if not crawled_page_links:+ return+ try:+ from mwmbl.models import SourceProvenance++ max_depth = getattr(settings, "SOURCE_PROVENANCE_MAX_DEPTH", 3)+ known = {+ url: (source, depth)+ for url, source, depth in SourceProvenance.objects.filter(+ url__in=list(crawled_page_links.keys())+ ).values_list("url", "source", "depth")+ }+ rows = []+ for page_url, link_urls in crawled_page_links.items():+ entry = known.get(page_url)+ if entry is None:+ continue+ source, depth = entry+ if depth >= max_depth:+ continue+ for link_url in link_urls:+ rows.append(SourceProvenance(+ url=link_url, source=source, parent_url=page_url[:500],+ depth=depth + 1, query=None,+ ))+ if rows:+ SourceProvenance.objects.bulk_create(rows, ignore_conflicts=True)+ logger.info(f"Propagated source provenance to {len(rows)} links")+ except Exception:+ logger.exception("failed to propagate source provenance")def get_datetime_from_timestamp(timestamp: float) -> datetime:diff --git a/mwmbl/migrations/0026_sourceprovenance.py b/mwmbl/migrations/0026_sourceprovenance.pynew file mode 100644index 0000000..a222d8f--- /dev/null+++ b/mwmbl/migrations/0026_sourceprovenance.py@@ -0,0 +1,28 @@+# Generated by Django 5.2.13 on 2026-06-28 21:16++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ('mwmbl', '0025_supersearchbanditstate'),+ ]++ operations = [+ migrations.CreateModel(+ name='SourceProvenance',+ fields=[+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),+ ('url', models.URLField(max_length=500, unique=True)),+ ('source', models.CharField(max_length=128)),+ ('query', models.CharField(blank=True, max_length=512, null=True)),+ ('parent_url', models.URLField(blank=True, max_length=500, null=True)),+ ('depth', models.IntegerField(default=0)),+ ('timestamp', models.DateTimeField(auto_now_add=True)),+ ],+ options={+ 'indexes': [models.Index(fields=['source'], name='mwmbl_sourc_source_e12d82_idx'), models.Index(fields=['timestamp'], name='mwmbl_sourc_timesta_bb15f3_idx')],+ },+ ),+ ]diff --git a/mwmbl/models.py b/mwmbl/models.pyindex bc5b395..33d2c98 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -260,3 +260,24 @@ class SuperSearchBanditState(models.Model):a = models.BinaryField()b = models.BinaryField()updated_at = models.DateTimeField(auto_now=True)+++class SourceProvenance(models.Model):+ """Which Super Search source a URL was (transitively) discovered from.++ Written for each URL a source returns, and propagated onto links found on+ pages later crawled from those URLs, so source usefulness can be judged+ offline including for descendant pages.+ """+ url = models.URLField(max_length=500, unique=True) # first source wins, matches source_by_url semantics+ source = models.CharField(max_length=128) # super-search source name (e.g. "gov.uk")+ query = models.CharField(max_length=512, null=True, blank=True) # query for direct results; null when propagated+ parent_url = models.URLField(max_length=500, null=True, blank=True) # page this URL was found on (null = direct result)+ depth = models.IntegerField(default=0) # 0 = direct super-search result; +1 per crawl hop+ timestamp = models.DateTimeField(auto_now_add=True)++ class Meta:+ indexes = [+ models.Index(fields=['source']),+ models.Index(fields=['timestamp']),+ ]diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 208334f..dedd11e 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -298,3 +298,4 @@ SUPER_SEARCH_TS_NOISE_VARIANCE = 0.25 # reward noise variance (sigma^2); rewardSUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectorsSUPER_SEARCH_USE_BANDIT = False # False = cosine-top-10 baseline; True = Thompson samplingSUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on+SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pagesdiff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 3d9db7d..89c88f7 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -15,6 +15,7 @@ import asyncioimport copyimport heapqfrom concurrent.futures import ThreadPoolExecutor+from datetime import datetimeimport jsonimport loggingimport re@@ -32,14 +33,16 @@ from ninja.errors import HttpErrorfrom pydantic import BaseModel, Fieldfrom mwmbl.crawler.retrieve import crawl_url+from mwmbl.crawler.urls import FoundURL, URLStatusfrom mwmbl.indexer.index_batches import index_results_against_query+from mwmbl.indexer.update_urls import _add_found_urls_to_db_and_queuefrom mwmbl.quota import (check_rate_limit,decrement_monthly_super_search,increment_monthly_super_search,)from mwmbl.search_auth import authenticate_user-from mwmbl.search_setup import index_path, ltr_model+from mwmbl.search_setup import index_path, ltr_model, queued_batchesfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.ltr_rank import score_documentsfrom mwmbl.tinysearchengine.mmr_rank import mmr_rerank@@ -52,6 +55,7 @@ from mwmbl.tinysearchengine.super_search_select.rewards import (SelectionContext,compute_rewards,log_impression,+ record_source_provenance,)from mwmbl.tokenizer import tokenize@@ -128,9 +132,10 @@ class ResultItem(Schema):examples=[1.8423])source: str = Field(description=(- "Originating source for `result_promoted` (one of the Super Search "- "sources, e.g. `github`); empty string for items in the final "- "`results` ranking, which merges all sources."+ "Originating Super Search source for this result (e.g. `github`) — "+ "the source that first produced the URL, for both `result_promoted` "+ "and final `results` items. Empty string when the source is unknown "+ "(e.g. an inline-followed link whose parent had no recorded source)."),examples=["github", ""],)@@ -363,10 +368,13 @@ async def _call_source(name: str, fn, client: httpx.AsyncClient, query: str, limasync def _follow_links(parent: Document, query: str, emit, all_docs: list[Document],- last_results_key: list, lock: asyncio.Lock,+ last_results_key: list, lock: asyncio.Lock, ctx: SelectionContext,) -> None:"""Crawl the parent URL, score its outbound links, and collect the best for final ranking."""max_links = settings.SUPER_SEARCH_MAX_LINKS_PER_PAGE+ # Inline-followed pages are surfaced by Super Search itself, so attribute+ # them (and the parent) to the source that produced the parent result.+ parent_source = ctx.source_by_url.get(parent.url)try:result = await _crawl(parent.url)@@ -388,7 +396,7 @@ async def _follow_links(all_docs.append(Document(title=parent_title, url=parent.url, extract=parent_extract))if not raw_links:- await _emit_final_results(query, all_docs, emit, last_results_key, lock)+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)returnterms = tokenize(query)@@ -421,12 +429,15 @@ async def _follow_links(url=proxy_doc.url,extract=c.get("extract") or "",))+ if parent_source:+ ctx.record_results(parent_source, [proxy_doc.url])- await _emit_final_results(query, all_docs, emit, last_results_key, lock)+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)async def _emit_final_results(- query: str, all_docs: list[Document], emit, last_results_key: list, lock: asyncio.Lock+ query: str, all_docs: list[Document], emit, last_results_key: list, lock: asyncio.Lock,+ ctx: SelectionContext,) -> None:terms = tokenize(query)final_limit = getattr(settings, "SUPER_SEARCH_FINAL_RESULTS_LIMIT", 100)@@ -461,7 +472,10 @@ async def _emit_final_results(returnlast_results_key[0] = keyawait emit("results", ResultsEvent(- results=[_result_payload(doc, score, "", "final") for doc, score in ranked],+ results=[+ _result_payload(doc, score, ctx.source_by_url.get(doc.url, ""), "final")+ for doc, score in ranked+ ],count=len(ranked),))@@ -483,6 +497,23 @@ async def _index_results(query: str, docs: list[Document]) -> int:return 0+def _enqueue_discovered_urls(ctx: SelectionContext) -> None:+ """Queue every URL Super Search discovered for organic crawling.++ Tags each FoundURL with its originating source name in the user_id_hash slot+ (consistent with the 'hn' seed precedent), so descendant pages crawled later+ can inherit the source via SourceProvenance. Blocking (Redis + Postgres);+ run off the event loop. Never raises into the caller.+ """+ now = datetime.utcnow()+ found_urls = [+ FoundURL(url, source, URLStatus.NEW, now, last_crawled=None)+ for url, source in ctx.source_by_url.items()+ ]+ if found_urls:+ _add_found_urls_to_db_and_queue(found_urls, queued_batches)++async def _record_rewards(query: str, ctx: SelectionContext, last_results_key: list) -> None:"""Compute implicit per-source rewards from the final top-K and log the impression."""if not ctx.selected:@@ -494,6 +525,14 @@ async def _record_rewards(query: str, ctx: SelectionContext, last_results_key: lawait asyncio.to_thread(log_impression, query, ctx, rewards)except Exception:logger.exception("super-search failed to record rewards")+ try:+ await asyncio.to_thread(record_source_provenance, query, ctx)+ except Exception:+ logger.exception("super-search failed to record source provenance")+ try:+ await asyncio.to_thread(_enqueue_discovered_urls, ctx)+ except Exception:+ logger.exception("super-search failed to enqueue discovered urls")if getattr(settings, "SUPER_SEARCH_USE_BANDIT", False):try:await asyncio.to_thread(ss_bandit.update, ctx.features, rewards)@@ -578,10 +617,10 @@ async def _run_pipeline(if _maybe_promote(doc, score):await emit("result_promoted", _result_payload(doc, score, name, "direct"))secondary.append(- asyncio.create_task(_follow_links(doc, query, emit, all_docs, last_results_key, lock))+ asyncio.create_task(_follow_links(doc, query, emit, all_docs, last_results_key, lock, ctx)))if all_docs:- await _emit_final_results(query, all_docs, emit, last_results_key, lock)+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)if secondary:results = await asyncio.gather(*secondary, return_exceptions=True)@@ -626,7 +665,7 @@ async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):finally:if reason in ("complete", "timed_out"):try:- await _emit_final_results(query, all_docs, emit, last_results_key, results_lock)+ await _emit_final_results(query, all_docs, emit, last_results_key, results_lock, selection_ctx)except Exception:logger.exception("super-search failed to emit final results")pages_indexed = await _index_results(query, all_docs)diff --git a/mwmbl/tinysearchengine/super_search_select/rewards.py b/mwmbl/tinysearchengine/super_search_select/rewards.pyindex c7b0270..58791c3 100644--- a/mwmbl/tinysearchengine/super_search_select/rewards.py+++ b/mwmbl/tinysearchengine/super_search_select/rewards.py@@ -64,3 +64,27 @@ def log_impression(query: str, ctx: SelectionContext, rewards: dict[str, float]))except Exception:logger.exception("failed to log super-search impression")+++def record_source_provenance(query: str, ctx: SelectionContext) -> None:+ """Persist a SourceProvenance row per (url, source) Super Search returned.++ Records the durable url -> source mapping (depth 0) so source usefulness can+ be judged offline, including for pages crawled later from these URLs. No-op+ without a database; conflicts on the unique url are ignored (first source to+ produce a URL wins, matching SelectionContext.record_results).+ """+ if not getattr(settings, "HAS_DATABASE", False):+ return+ if not ctx.source_by_url:+ return+ try:+ from mwmbl.models import SourceProvenance++ rows = [+ SourceProvenance(url=url, source=source, query=query[:512], depth=0)+ for url, source in ctx.source_by_url.items()+ ]+ SourceProvenance.objects.bulk_create(rows, ignore_conflicts=True)+ except Exception:+ logger.exception("failed to record super-search source provenance")diff --git a/test/test_super_search.py b/test/test_super_search.pyindex 24de687..39681a4 100644--- a/test/test_super_search.py+++ b/test/test_super_search.py@@ -303,6 +303,35 @@ def test_final_results_event_emitted(client, api_key, monkeypatch):assert event_types.index("results") < event_types.index("done")+def test_final_results_carry_originating_source(client, api_key, monkeypatch):+ """Final 'results' items report the source that produced their URL (not '')."""+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "github": [Document(title="Python repo", url="https://gh.example/", extract="python code")],+ "hn": [Document(title="Python thread", url="https://hn.example/", extract="python talk")],+ })+ _stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 20)++ def fake_crawl(url, redis):+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get(+ "/api/v2/super-search/?q=python",+ HTTP_X_API_KEY=api_key.raw_key,+ )+ assert response.status_code == 200+ events = _parse_sse(_read_stream(response))+ final = [d for t, d in events if t == "results"][-1]["results"]+ source_by_url = {item["url"]: item["source"] for item in final}+ assert source_by_url["https://gh.example/"] == "github"+ assert source_by_url["https://hn.example/"] == "hn"+ # Every final item is tagged origin=final.+ assert all(item["origin"] == "final" for item in final)++@pytest.mark.django_dbdef test_final_results_are_mmr_diversified(client, api_key, monkeypatch):"""The final ranking applies MMR diversity (as standard search does): a seconddiff --git a/test/test_super_search_select.py b/test/test_super_search_select.pyindex 5988ebd..c35e5f6 100644--- a/test/test_super_search_select.py+++ b/test/test_super_search_select.py@@ -173,3 +173,34 @@ def test_log_impression_noop_without_database(monkeypatch):monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)# Should not raise even though no DB is configured.rewards.log_impression("q", SelectionContext(selected=["a"]), {"a": 0.5})+++def test_record_source_provenance_noop_without_database(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)+ ctx = SelectionContext(selected=["a"])+ ctx.record_results("a", ["https://x/u1"])+ # Should not raise even though provenance is gated off.+ rewards.record_source_provenance("q", ctx)+++def test_record_source_provenance_writes_direct_rows(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)+ from mwmbl.models import SourceProvenance++ ctx = SelectionContext(selected=["a", "b"])+ ctx.record_results("a", ["https://x/u1", "https://x/u2"])+ ctx.record_results("b", ["https://x/u3"])++ rewards.record_source_provenance("my query", ctx)++ assert SourceProvenance.objects.count() == 3+ row = SourceProvenance.objects.get(url="https://x/u1")+ assert row.source == "a"+ assert row.depth == 0+ assert row.query == "my query"+ assert row.parent_url is None++ # Idempotent: a second call must not raise or duplicate (unique url + ignore_conflicts).+ rewards.record_source_provenance("my query", ctx)+ assert SourceProvenance.objects.count() == 3diff --git a/test/test_update_urls.py b/test/test_update_urls.pyindex 47894c6..e143355 100644--- a/test/test_update_urls.py+++ b/test/test_update_urls.py@@ -1,15 +1,18 @@from collections import defaultdict+import pytest+from mwmbl.crawler.batch import Link-from mwmbl.indexer.update_urls import process_link+from mwmbl.indexer.update_urls import process_link, _propagate_source_provenancefrom mwmbl.indexer.blacklist_providers import StaticBlacklistProvider+from mwmbl.models import SourceProvenancedef test_process_link_normal():- url_scores = {"https://somesite.com/something.html": 0.0, "https://somesite.com/": 0.0}url_timestamps = {}url_users = {}domain_links = defaultdict(set)+ crawled_page_links = {}blacklist_provider = StaticBlacklistProvider(set())process_link(@@ -21,16 +24,22 @@ def test_process_link_normal():url_users=url_users,blacklist_provider=blacklist_provider,domain_links=domain_links,+ page_url="https://somewhere.com/page.html",+ crawled_page_links=crawled_page_links,)assert domain_links == {"somewhere.com": {"somesite.com"}}+ # The accepted link is remembered against the page it was found on.+ assert crawled_page_links == {+ "https://somewhere.com/page.html": ["https://somesite.com/something.html"]+ }def test_process_link_excludes_porn():- url_scores = {}url_timestamps = {}url_users = {}domain_links = {}+ crawled_page_links = {}# Create a blacklist provider that blocks porn sitesblacklist_provider = StaticBlacklistProvider({"somepornsite.com"})@@ -43,9 +52,57 @@ def test_process_link_excludes_porn():url_users=url_users,blacklist_provider=blacklist_provider,domain_links=domain_links,+ page_url="https://somewhere.com/page.html",+ crawled_page_links=crawled_page_links,)- assert url_scores == {}assert url_timestamps == {}assert url_users == {}assert domain_links == {}+ # Blacklisted links are not recorded for provenance propagation either.+ assert crawled_page_links == {}+++PAGE = "https://gov.uk/guidance"+CHILD_1 = "https://gov.uk/guidance/details"+CHILD_2 = "https://example.com/referenced"+++def test_propagate_source_provenance_noop_without_database(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)+ # Should not raise even though provenance is gated off.+ _propagate_source_provenance({PAGE: [CHILD_1]})+++def test_propagate_source_provenance_carries_source_to_links(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)+ SourceProvenance.objects.create(url=PAGE, source="gov.uk", query="benefits", depth=0)++ _propagate_source_provenance({PAGE: [CHILD_1, CHILD_2]})++ for child in (CHILD_1, CHILD_2):+ row = SourceProvenance.objects.get(url=child)+ assert row.source == "gov.uk"+ assert row.parent_url == PAGE+ assert row.depth == 1+ assert row.query is None+++def test_propagate_source_provenance_skips_unknown_pages(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)+ # PAGE has no provenance row, so nothing should be written for its links.+ _propagate_source_provenance({PAGE: [CHILD_1, CHILD_2]})+ assert SourceProvenance.objects.count() == 0+++def test_propagate_source_provenance_respects_depth_cap(monkeypatch):+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)+ monkeypatch.setattr("django.conf.settings.SOURCE_PROVENANCE_MAX_DEPTH", 2)+ SourceProvenance.objects.create(url=PAGE, source="gov.uk", depth=2)++ _propagate_source_provenance({PAGE: [CHILD_1]})++ assert not SourceProvenance.objects.filter(url=CHILD_1).exists()