Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search: track per-URL source provenance + propagate to crawls

Daoud Clarkec293a7812c9a
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index bdd2125..9181cec 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -50,6 +50,7 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Re
url_timestamps = {}
url_statuses = defaultdict(lambda: URLStatus.NEW)
domain_links = defaultdict(set)
+ crawled_page_links: dict[str, list[str]] = {}
for batch in batches:
for item in batch.items:
timestamp = get_datetime_from_timestamp(item.timestamp / 1000.0)
@@ -66,7 +67,7 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Re
continue
for link in item.content.all_links:
process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,
- blacklist_provider, domain_links)
+ blacklist_provider, domain_links, item.url, crawled_page_links)
found_urls = [FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])
for url in url_statuses.keys() | url_users.keys()]
@@ -79,6 +80,8 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Re
for source_domain, target_domains in domain_links.items():
domain_link_db.update_domain_links(source_domain, target_domains)
+ _propagate_source_provenance(crawled_page_links)
+
end = datetime.utcnow()
logger.info(f"Recorded URLs in database in {end - start}")
@@ -108,7 +111,7 @@ def add_hn_links(new_item_queue: RedisURLQueue):
def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timestamp: datetime,
url_timestamps: dict[str, datetime], url_users: dict[str, str], blacklist_provider,
- domain_links: dict[str, set[str]]):
+ domain_links: dict[str, set[str]], page_url: str, crawled_page_links: dict[str, list[str]]):
try:
parsed_link = parse_url(link.url)
except ValueError:
@@ -129,6 +132,52 @@ def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timest
url_users[root_url] = user_id_hash
url_timestamps[root_url] = timestamp
domain_links[crawled_page_domain].add(parsed_link.netloc)
+ # Remember which links came off which crawled page, for source-provenance
+ # propagation (only used when the page itself has a known Super Search source).
+ if len(link.url) <= 500:
+ crawled_page_links.setdefault(page_url, []).append(link.url)
+
+
+def _propagate_source_provenance(crawled_page_links: dict[str, list[str]]):
+ """Carry a crawled page's Super Search source onto the links found on it.
+
+ Only pages already recorded in SourceProvenance propagate, so the common case
+ (no provenance) costs a single batched query. Propagation is bounded by
+ SOURCE_PROVENANCE_MAX_DEPTH to limit transitive spread. No-op without a
+ database; never raises into URL recording.
+ """
+ if not getattr(settings, "HAS_DATABASE", False):
+ return
+ if not crawled_page_links:
+ return
+ try:
+ from mwmbl.models import SourceProvenance
+
+ max_depth = getattr(settings, "SOURCE_PROVENANCE_MAX_DEPTH", 3)
+ known = {
+ url: (source, depth)
+ for url, source, depth in SourceProvenance.objects.filter(
+ url__in=list(crawled_page_links.keys())
+ ).values_list("url", "source", "depth")
+ }
+ rows = []
+ for page_url, link_urls in crawled_page_links.items():
+ entry = known.get(page_url)
+ if entry is None:
+ continue
+ source, depth = entry
+ if depth >= max_depth:
+ continue
+ for link_url in link_urls:
+ rows.append(SourceProvenance(
+ url=link_url, source=source, parent_url=page_url[:500],
+ depth=depth + 1, query=None,
+ ))
+ if rows:
+ SourceProvenance.objects.bulk_create(rows, ignore_conflicts=True)
+ logger.info(f"Propagated source provenance to {len(rows)} links")
+ except Exception:
+ logger.exception("failed to propagate source provenance")
def get_datetime_from_timestamp(timestamp: float) -> datetime:
diff --git a/mwmbl/migrations/0026_sourceprovenance.py b/mwmbl/migrations/0026_sourceprovenance.py
new file mode 100644
index 0000000..a222d8f
--- /dev/null
+++ b/mwmbl/migrations/0026_sourceprovenance.py
@@ -0,0 +1,28 @@
+# Generated by Django 5.2.13 on 2026-06-28 21:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('mwmbl', '0025_supersearchbanditstate'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='SourceProvenance',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('url', models.URLField(max_length=500, unique=True)),
+ ('source', models.CharField(max_length=128)),
+ ('query', models.CharField(blank=True, max_length=512, null=True)),
+ ('parent_url', models.URLField(blank=True, max_length=500, null=True)),
+ ('depth', models.IntegerField(default=0)),
+ ('timestamp', models.DateTimeField(auto_now_add=True)),
+ ],
+ options={
+ 'indexes': [models.Index(fields=['source'], name='mwmbl_sourc_source_e12d82_idx'), models.Index(fields=['timestamp'], name='mwmbl_sourc_timesta_bb15f3_idx')],
+ },
+ ),
+ ]
diff --git a/mwmbl/models.py b/mwmbl/models.py
index bc5b395..33d2c98 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -260,3 +260,24 @@ class SuperSearchBanditState(models.Model):
a = models.BinaryField()
b = models.BinaryField()
updated_at = models.DateTimeField(auto_now=True)
+
+
+class SourceProvenance(models.Model):
+ """Which Super Search source a URL was (transitively) discovered from.
+
+ Written for each URL a source returns, and propagated onto links found on
+ pages later crawled from those URLs, so source usefulness can be judged
+ offline including for descendant pages.
+ """
+ url = models.URLField(max_length=500, unique=True) # first source wins, matches source_by_url semantics
+ source = models.CharField(max_length=128) # super-search source name (e.g. "gov.uk")
+ query = models.CharField(max_length=512, null=True, blank=True) # query for direct results; null when propagated
+ parent_url = models.URLField(max_length=500, null=True, blank=True) # page this URL was found on (null = direct result)
+ depth = models.IntegerField(default=0) # 0 = direct super-search result; +1 per crawl hop
+ timestamp = models.DateTimeField(auto_now_add=True)
+
+ class Meta:
+ indexes = [
+ models.Index(fields=['source']),
+ models.Index(fields=['timestamp']),
+ ]
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index 208334f..dedd11e 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -298,3 +298,4 @@ SUPER_SEARCH_TS_NOISE_VARIANCE = 0.25 # reward noise variance (sigma^2); reward
SUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectors
SUPER_SEARCH_USE_BANDIT = False # False = cosine-top-10 baseline; True = Thompson sampling
SUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on
+SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.py
index 3d9db7d..89c88f7 100644
--- a/mwmbl/tinysearchengine/super_search.py
+++ b/mwmbl/tinysearchengine/super_search.py
@@ -15,6 +15,7 @@ import asyncio
import copy
import heapq
from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
import json
import logging
import re
@@ -32,14 +33,16 @@ from ninja.errors import HttpError
from pydantic import BaseModel, Field
from mwmbl.crawler.retrieve import crawl_url
+from mwmbl.crawler.urls import FoundURL, URLStatus
from mwmbl.indexer.index_batches import index_results_against_query
+from mwmbl.indexer.update_urls import _add_found_urls_to_db_and_queue
from mwmbl.quota import (
check_rate_limit,
decrement_monthly_super_search,
increment_monthly_super_search,
)
from mwmbl.search_auth import authenticate_user
-from mwmbl.search_setup import index_path, ltr_model
+from mwmbl.search_setup import index_path, ltr_model, queued_batches
from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.ltr_rank import score_documents
from mwmbl.tinysearchengine.mmr_rank import mmr_rerank
@@ -52,6 +55,7 @@ from mwmbl.tinysearchengine.super_search_select.rewards import (
SelectionContext,
compute_rewards,
log_impression,
+ record_source_provenance,
)
from mwmbl.tokenizer import tokenize
@@ -128,9 +132,10 @@ class ResultItem(Schema):
examples=[1.8423])
source: str = Field(
description=(
- "Originating source for `result_promoted` (one of the Super Search "
- "sources, e.g. `github`); empty string for items in the final "
- "`results` ranking, which merges all sources."
+ "Originating Super Search source for this result (e.g. `github`) — "
+ "the source that first produced the URL, for both `result_promoted` "
+ "and final `results` items. Empty string when the source is unknown "
+ "(e.g. an inline-followed link whose parent had no recorded source)."
),
examples=["github", ""],
)
@@ -363,10 +368,13 @@ async def _call_source(name: str, fn, client: httpx.AsyncClient, query: str, lim
async def _follow_links(
parent: Document, query: str, emit, all_docs: list[Document],
- last_results_key: list, lock: asyncio.Lock,
+ last_results_key: list, lock: asyncio.Lock, ctx: SelectionContext,
) -> None:
"""Crawl the parent URL, score its outbound links, and collect the best for final ranking."""
max_links = settings.SUPER_SEARCH_MAX_LINKS_PER_PAGE
+ # Inline-followed pages are surfaced by Super Search itself, so attribute
+ # them (and the parent) to the source that produced the parent result.
+ parent_source = ctx.source_by_url.get(parent.url)
try:
result = await _crawl(parent.url)
@@ -388,7 +396,7 @@ async def _follow_links(
all_docs.append(Document(title=parent_title, url=parent.url, extract=parent_extract))
if not raw_links:
- await _emit_final_results(query, all_docs, emit, last_results_key, lock)
+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)
return
terms = tokenize(query)
@@ -421,12 +429,15 @@ async def _follow_links(
url=proxy_doc.url,
extract=c.get("extract") or "",
))
+ if parent_source:
+ ctx.record_results(parent_source, [proxy_doc.url])
- await _emit_final_results(query, all_docs, emit, last_results_key, lock)
+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)
async def _emit_final_results(
- query: str, all_docs: list[Document], emit, last_results_key: list, lock: asyncio.Lock
+ query: str, all_docs: list[Document], emit, last_results_key: list, lock: asyncio.Lock,
+ ctx: SelectionContext,
) -> None:
terms = tokenize(query)
final_limit = getattr(settings, "SUPER_SEARCH_FINAL_RESULTS_LIMIT", 100)
@@ -461,7 +472,10 @@ async def _emit_final_results(
return
last_results_key[0] = key
await emit("results", ResultsEvent(
- results=[_result_payload(doc, score, "", "final") for doc, score in ranked],
+ results=[
+ _result_payload(doc, score, ctx.source_by_url.get(doc.url, ""), "final")
+ for doc, score in ranked
+ ],
count=len(ranked),
))
@@ -483,6 +497,23 @@ async def _index_results(query: str, docs: list[Document]) -> int:
return 0
+def _enqueue_discovered_urls(ctx: SelectionContext) -> None:
+ """Queue every URL Super Search discovered for organic crawling.
+
+ Tags each FoundURL with its originating source name in the user_id_hash slot
+ (consistent with the 'hn' seed precedent), so descendant pages crawled later
+ can inherit the source via SourceProvenance. Blocking (Redis + Postgres);
+ run off the event loop. Never raises into the caller.
+ """
+ now = datetime.utcnow()
+ found_urls = [
+ FoundURL(url, source, URLStatus.NEW, now, last_crawled=None)
+ for url, source in ctx.source_by_url.items()
+ ]
+ if found_urls:
+ _add_found_urls_to_db_and_queue(found_urls, queued_batches)
+
+
async def _record_rewards(query: str, ctx: SelectionContext, last_results_key: list) -> None:
"""Compute implicit per-source rewards from the final top-K and log the impression."""
if not ctx.selected:
@@ -494,6 +525,14 @@ async def _record_rewards(query: str, ctx: SelectionContext, last_results_key: l
await asyncio.to_thread(log_impression, query, ctx, rewards)
except Exception:
logger.exception("super-search failed to record rewards")
+ try:
+ await asyncio.to_thread(record_source_provenance, query, ctx)
+ except Exception:
+ logger.exception("super-search failed to record source provenance")
+ try:
+ await asyncio.to_thread(_enqueue_discovered_urls, ctx)
+ except Exception:
+ logger.exception("super-search failed to enqueue discovered urls")
if getattr(settings, "SUPER_SEARCH_USE_BANDIT", False):
try:
await asyncio.to_thread(ss_bandit.update, ctx.features, rewards)
@@ -578,10 +617,10 @@ async def _run_pipeline(
if _maybe_promote(doc, score):
await emit("result_promoted", _result_payload(doc, score, name, "direct"))
secondary.append(
- asyncio.create_task(_follow_links(doc, query, emit, all_docs, last_results_key, lock))
+ asyncio.create_task(_follow_links(doc, query, emit, all_docs, last_results_key, lock, ctx))
)
if all_docs:
- await _emit_final_results(query, all_docs, emit, last_results_key, lock)
+ await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)
if secondary:
results = await asyncio.gather(*secondary, return_exceptions=True)
@@ -626,7 +665,7 @@ async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):
finally:
if reason in ("complete", "timed_out"):
try:
- await _emit_final_results(query, all_docs, emit, last_results_key, results_lock)
+ await _emit_final_results(query, all_docs, emit, last_results_key, results_lock, selection_ctx)
except Exception:
logger.exception("super-search failed to emit final results")
pages_indexed = await _index_results(query, all_docs)
diff --git a/mwmbl/tinysearchengine/super_search_select/rewards.py b/mwmbl/tinysearchengine/super_search_select/rewards.py
index c7b0270..58791c3 100644
--- a/mwmbl/tinysearchengine/super_search_select/rewards.py
+++ b/mwmbl/tinysearchengine/super_search_select/rewards.py
@@ -64,3 +64,27 @@ def log_impression(query: str, ctx: SelectionContext, rewards: dict[str, float])
)
except Exception:
logger.exception("failed to log super-search impression")
+
+
+def record_source_provenance(query: str, ctx: SelectionContext) -> None:
+ """Persist a SourceProvenance row per (url, source) Super Search returned.
+
+ Records the durable url -> source mapping (depth 0) so source usefulness can
+ be judged offline, including for pages crawled later from these URLs. No-op
+ without a database; conflicts on the unique url are ignored (first source to
+ produce a URL wins, matching SelectionContext.record_results).
+ """
+ if not getattr(settings, "HAS_DATABASE", False):
+ return
+ if not ctx.source_by_url:
+ return
+ try:
+ from mwmbl.models import SourceProvenance
+
+ rows = [
+ SourceProvenance(url=url, source=source, query=query[:512], depth=0)
+ for url, source in ctx.source_by_url.items()
+ ]
+ SourceProvenance.objects.bulk_create(rows, ignore_conflicts=True)
+ except Exception:
+ logger.exception("failed to record super-search source provenance")
diff --git a/test/test_super_search.py b/test/test_super_search.py
index 24de687..39681a4 100644
--- a/test/test_super_search.py
+++ b/test/test_super_search.py
@@ -303,6 +303,35 @@ def test_final_results_event_emitted(client, api_key, monkeypatch):
assert event_types.index("results") < event_types.index("done")
+def test_final_results_carry_originating_source(client, api_key, monkeypatch):
+ """Final 'results' items report the source that produced their URL (not '')."""
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "github": [Document(title="Python repo", url="https://gh.example/", extract="python code")],
+ "hn": [Document(title="Python thread", url="https://hn.example/", extract="python talk")],
+ })
+ _stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 20)
+
+ def fake_crawl(url, redis):
+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get(
+ "/api/v2/super-search/?q=python",
+ HTTP_X_API_KEY=api_key.raw_key,
+ )
+ assert response.status_code == 200
+ events = _parse_sse(_read_stream(response))
+ final = [d for t, d in events if t == "results"][-1]["results"]
+ source_by_url = {item["url"]: item["source"] for item in final}
+ assert source_by_url["https://gh.example/"] == "github"
+ assert source_by_url["https://hn.example/"] == "hn"
+ # Every final item is tagged origin=final.
+ assert all(item["origin"] == "final" for item in final)
+
+
@pytest.mark.django_db
def test_final_results_are_mmr_diversified(client, api_key, monkeypatch):
"""The final ranking applies MMR diversity (as standard search does): a second
diff --git a/test/test_super_search_select.py b/test/test_super_search_select.py
index 5988ebd..c35e5f6 100644
--- a/test/test_super_search_select.py
+++ b/test/test_super_search_select.py
@@ -173,3 +173,34 @@ def test_log_impression_noop_without_database(monkeypatch):
monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)
# Should not raise even though no DB is configured.
rewards.log_impression("q", SelectionContext(selected=["a"]), {"a": 0.5})
+
+
+def test_record_source_provenance_noop_without_database(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)
+ ctx = SelectionContext(selected=["a"])
+ ctx.record_results("a", ["https://x/u1"])
+ # Should not raise even though provenance is gated off.
+ rewards.record_source_provenance("q", ctx)
+
+
+def test_record_source_provenance_writes_direct_rows(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)
+ from mwmbl.models import SourceProvenance
+
+ ctx = SelectionContext(selected=["a", "b"])
+ ctx.record_results("a", ["https://x/u1", "https://x/u2"])
+ ctx.record_results("b", ["https://x/u3"])
+
+ rewards.record_source_provenance("my query", ctx)
+
+ assert SourceProvenance.objects.count() == 3
+ row = SourceProvenance.objects.get(url="https://x/u1")
+ assert row.source == "a"
+ assert row.depth == 0
+ assert row.query == "my query"
+ assert row.parent_url is None
+
+ # Idempotent: a second call must not raise or duplicate (unique url + ignore_conflicts).
+ rewards.record_source_provenance("my query", ctx)
+ assert SourceProvenance.objects.count() == 3
diff --git a/test/test_update_urls.py b/test/test_update_urls.py
index 47894c6..e143355 100644
--- a/test/test_update_urls.py
+++ b/test/test_update_urls.py
@@ -1,15 +1,18 @@
from collections import defaultdict
+import pytest
+
from mwmbl.crawler.batch import Link
-from mwmbl.indexer.update_urls import process_link
+from mwmbl.indexer.update_urls import process_link, _propagate_source_provenance
from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
+from mwmbl.models import SourceProvenance
def test_process_link_normal():
- url_scores = {"https://somesite.com/something.html": 0.0, "https://somesite.com/": 0.0}
url_timestamps = {}
url_users = {}
domain_links = defaultdict(set)
+ crawled_page_links = {}
blacklist_provider = StaticBlacklistProvider(set())
process_link(
@@ -21,16 +24,22 @@ def test_process_link_normal():
url_users=url_users,
blacklist_provider=blacklist_provider,
domain_links=domain_links,
+ page_url="https://somewhere.com/page.html",
+ crawled_page_links=crawled_page_links,
)
assert domain_links == {"somewhere.com": {"somesite.com"}}
+ # The accepted link is remembered against the page it was found on.
+ assert crawled_page_links == {
+ "https://somewhere.com/page.html": ["https://somesite.com/something.html"]
+ }
def test_process_link_excludes_porn():
- url_scores = {}
url_timestamps = {}
url_users = {}
domain_links = {}
+ crawled_page_links = {}
# Create a blacklist provider that blocks porn sites
blacklist_provider = StaticBlacklistProvider({"somepornsite.com"})
@@ -43,9 +52,57 @@ def test_process_link_excludes_porn():
url_users=url_users,
blacklist_provider=blacklist_provider,
domain_links=domain_links,
+ page_url="https://somewhere.com/page.html",
+ crawled_page_links=crawled_page_links,
)
- assert url_scores == {}
assert url_timestamps == {}
assert url_users == {}
assert domain_links == {}
+ # Blacklisted links are not recorded for provenance propagation either.
+ assert crawled_page_links == {}
+
+
+PAGE = "https://gov.uk/guidance"
+CHILD_1 = "https://gov.uk/guidance/details"
+CHILD_2 = "https://example.com/referenced"
+
+
+def test_propagate_source_provenance_noop_without_database(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)
+ # Should not raise even though provenance is gated off.
+ _propagate_source_provenance({PAGE: [CHILD_1]})
+
+
+def test_propagate_source_provenance_carries_source_to_links(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)
+ SourceProvenance.objects.create(url=PAGE, source="gov.uk", query="benefits", depth=0)
+
+ _propagate_source_provenance({PAGE: [CHILD_1, CHILD_2]})
+
+ for child in (CHILD_1, CHILD_2):
+ row = SourceProvenance.objects.get(url=child)
+ assert row.source == "gov.uk"
+ assert row.parent_url == PAGE
+ assert row.depth == 1
+ assert row.query is None
+
+
+def test_propagate_source_provenance_skips_unknown_pages(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)
+ # PAGE has no provenance row, so nothing should be written for its links.
+ _propagate_source_provenance({PAGE: [CHILD_1, CHILD_2]})
+ assert SourceProvenance.objects.count() == 0
+
+
+def test_propagate_source_provenance_respects_depth_cap(monkeypatch):
+ monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)
+ monkeypatch.setattr("django.conf.settings.SOURCE_PROVENANCE_MAX_DEPTH", 2)
+ SourceProvenance.objects.create(url=PAGE, source="gov.uk", depth=2)
+
+ _propagate_source_provenance({PAGE: [CHILD_1]})
+
+ assert not SourceProvenance.objects.filter(url=CHILD_1).exists()