An open source, non-profit web search engine
Cap seed crawls per domain and run them in a worker of their own
Claude3371bc033cba
diff --git a/mwmbl/background.py b/mwmbl/background.pyindex 6d93193..70f313f 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -6,7 +6,6 @@ Django Background Tasks for periodic maintenance:- report_usage_to_polar: reports billable usage overage to Polar once per hour- refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against- purge_blacklisted_from_queue: removes retrieval-filtered documents from the index- - seed_crawl: crawls outwards from a Combined Search's Staan results, on request"""import logging@@ -28,7 +27,6 @@ from mwmbl.crawler.stats import StatsManagerfrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist, refresh_snapshotfrom mwmbl.indexer.purge_blacklisted import purge_documentsfrom mwmbl.indexer.purge_queue import drain_purge_queue, queue_size-from mwmbl.indexer.seed_crawl import run_seed_crawlfrom mwmbl.models import DomainEvidence, DomainSubmission, OldIndex, UsageBucketfrom mwmbl.moderation.evidence import crawl_domain, store_evidence, store_failurefrom mwmbl.moderation.model import reset_model_cache@@ -253,16 +251,6 @@ def report_usage_to_polar():# ----------------------------------------------------------------------------@background(schedule=0)-def seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list[str]):- """Crawl the pages a Combined Search found that the index lacked - see mwmbl.indexer.seed_crawl.-- Takes minutes, which is why the queue runs tasks concurrently (BACKGROUND_TASK_RUN_ASYNC).- """- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME- run_seed_crawl(user_id, query, seed_urls, domains, str(index_path))--@background(schedule=0)def enrich_domain_submission(domain: str):"""Crawl a submitted domain and store the suggestion the moderation queue will read.diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.pyindex 8ced6cd..f0ec2e7 100644--- a/mwmbl/indexer/seed_crawl.py+++ b/mwmbl/indexer/seed_crawl.py@@ -1,14 +1,21 @@"""Seed crawls: crawling outwards from the pages Staan found that the index lacked.-Combined Search with crawl=true hands the Staan results the index did not already hold to-a background task (mwmbl.background.seed_crawl), which crawls them and follows their links,-breadth first, until it has fetched SEED_CRAWL_MAX_PAGES pages. Links are only followed to-the domains Staan returned for the query, so a crawl stays on the sites the query was about.+Combined Search with crawl=true queues the Staan results the index did not already hold,+and a worker process of its own (run_seed_crawl_worker, started by mwmbl.main) crawls them+and follows their links, breadth first. Links are only followed to the domains Staan+returned for the query, so a crawl stays on the sites the query was about.The crawl goes a round at a time, one URL per domain per round and at leastSEED_CRAWL_DOMAIN_DELAY_SECONDS apart, so no site is fetched from faster than that however-much of the frontier it holds. Each round is indexed as it finishes, so what a crawl has-added is visible while it is still running.+much of the frontier it holds. Each domain gives at most SEED_CRAWL_MAX_PAGES_PER_DOMAIN+pages, which also bounds a crawl to that many rounds - a few minutes - whatever the number+of domains. Each round is indexed as it finishes, so what a crawl has added is visible while+it is still running.++The crawls run in their own process rather than on the django-background-tasks queue: they+take minutes, and in series with the periodic tasks they would hold up the usage sync and+the Polar report. That process runs them one at a time, each user has at most one queued or+running, and at most SEED_CRAWL_MAX_QUEUED wait, so nobody can fill the queue.What it added is recorded in Redis against the user and the query, whereGET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and counted on the user for their stats.@@ -17,11 +24,13 @@ GET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and countimport hashlibimport jsonimport time-from collections import deque+from collections import Counter, dequefrom datetime import datetime, timezonefrom logging import getLogger+from pathlib import Pathfrom django.conf import settings+from django.db import close_old_connectionsfrom django.db.models import Ffrom django_redis import get_redis_connection@@ -34,8 +43,14 @@ from mwmbl.utils import bare_hostlogger = getLogger(__name__)+STATUS_QUEUED = "queued"STATUS_CRAWLING = "crawling"STATUS_DONE = "done"+STATUS_FAILED = "failed"++QUEUE_KEY = "seed-crawl:queue"+# Under the Redis client's five-second socket timeout, which a longer blocking pop would trip.+QUEUE_POLL_SECONDS = 1def _record_key(user_id: int, query: str) -> str:@@ -48,54 +63,118 @@ def _pages_key(user_id: int, query: str) -> str:return f"{_record_key(user_id, query)}:pages"+def _counted_urls_key(user_id: int, query: str) -> str:+ return f"{_record_key(user_id, query)}:urls"+++def _active_key(user_id: int) -> str:+ return f"seed-crawl:{user_id}:active"++def _now() -> str:return datetime.now(timezone.utc).isoformat()-def start_seed_crawl(user_id: int, query: str, index_pages: list[Document], staan_results: list[Document]):- """Record a seed crawl as started, returning the seed URLs and domains to crawl it with.+def start_seed_crawl(user_id: int, query: str, index_pages: list[Document], staan_results: list[Document]) -> bool:+ """Queue a seed crawl of the Staan results the index lacked, returning whether it did.- None when there is nothing to crawl - every Staan result is already in the index - or- when this user's crawl for this query is still running, which a repeated search must- not start again.+ Nothing is queued when every Staan result is already in the index, when this user+ already has a crawl queued or running, or when the queue is full."""blacklisted_urls = find_blacklisted_urls(staan_results)allowed = [document for document in staan_results if document.url not in blacklisted_urls]indexed_urls = {document.url for document in index_pages}seed_urls = [document.url for document in allowed if document.url not in indexed_urls]if not seed_urls:- return None+ return Falseredis = get_redis_connection("default")- record_key = _record_key(user_id, query)- if redis.hget(record_key, "status") == STATUS_CRAWLING.encode():- return None+ # A slight overshoot from racing requests is harmless; the cap is there to stop one+ # user, or many, queueing hours of crawling.+ if redis.llen(QUEUE_KEY) >= settings.SEED_CRAWL_MAX_QUEUED:+ return False+ # Set atomically, so a double click or a retried request cannot queue the crawl twice.+ # The expiry outlasts the longest wait plus the crawl, and frees the user should the+ # worker die mid-crawl.+ lock_seconds = settings.SEED_CRAWL_MAX_SECONDS * (settings.SEED_CRAWL_MAX_QUEUED + 1)+ if not redis.set(_active_key(user_id), query, nx=True, ex=lock_seconds):+ return False- ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS+ record_key = _record_key(user_id, query)+ domains = sorted({bare_host(document.url) for document in allowed})+ job = {"user_id": user_id, "query": query, "seed_urls": seed_urls, "domains": domains}+ # An earlier crawl's pages are kept: the new one adds to them, and the user's total+ # already includes them.pipeline = redis.pipeline()- pipeline.delete(record_key, _pages_key(user_id, query))- pipeline.hset(- record_key,- mapping={"status": STATUS_CRAWLING, "query": query, "started_at": _now(), "pages_crawled": 0},- )- pipeline.expire(record_key, ttl)+ pipeline.hset(record_key, mapping={"status": STATUS_QUEUED, "query": query, "started_at": _now()})+ pipeline.hsetnx(record_key, "pages_crawled", 0)+ pipeline.hdel(record_key, "finished_at")+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)+ pipeline.lpush(QUEUE_KEY, json.dumps(job))pipeline.execute()+ return True- domains = sorted({bare_host(document.url) for document in allowed})- return seed_urls, domains++def run_seed_crawl_worker() -> None:+ """Run queued seed crawls, one at a time, for ever."""+ index_path = str(Path(settings.DATA_PATH) / settings.INDEX_NAME)+ while True:+ run_next_seed_crawl(index_path)+++def run_next_seed_crawl(index_path: str) -> bool:+ """Run the next queued seed crawl, if one arrives within QUEUE_POLL_SECONDS."""+ redis = get_redis_connection("default")+ popped = redis.brpop(QUEUE_KEY, timeout=QUEUE_POLL_SECONDS)+ if popped is None:+ return False+ # A worker that waits for hours between crawls would otherwise reuse a connection+ # Postgres has long since closed.+ close_old_connections()+ job = json.loads(popped[1])+ run_seed_crawl(job["user_id"], job["query"], job["seed_urls"], job["domains"], index_path)+ return Truedef run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list[str], index_path: str) -> None:redis = get_redis_connection("default")+ record_key = _record_key(user_id, query)+ redis.hset(record_key, "status", STATUS_CRAWLING)+ try:+ _crawl_and_record(user_id, query, seed_urls, domains, index_path, redis)+ status = STATUS_DONE+ except Exception:+ # Recorded rather than retried: a retry would start again from the seeds, fetching+ # every site a second time. The worker carries on with the next crawl.+ logger.exception("Seed crawl for user %d failed", user_id)+ status = STATUS_FAILED+ finally:+ redis.delete(_active_key(user_id))++ pipeline = redis.pipeline()+ pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)+ pipeline.execute()+++def _crawl_and_record(user_id: int, query: str, seed_urls: list[str], domains: list[str], index_path: str, redis):record_key = _record_key(user_id, query)pages_key = _pages_key(user_id, query)+ counted_urls_key = _counted_urls_key(user_id, query)ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDSfor num_crawled, documents in crawl_within_domains(seed_urls, set(domains), redis):# The seeds were missing from the index when the user searched, but Combined Search# has since written Staan's snippets of them, so the index alone no longer says so.new_urls = index_new_documents(documents, index_path) | set(seed_urls)- new_documents = [document for document in documents if document.url in new_urls]+ candidates = [document for document in documents if document.url in new_urls]++ # A page is counted once per query, however many crawls of it find it new.+ pipeline = redis.pipeline()+ for document in candidates:+ pipeline.sadd(counted_urls_key, document.url)+ first_counted = pipeline.execute()+ new_documents = [document for document, added in zip(candidates, first_counted) if added]logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))pipeline = redis.pipeline()@@ -104,6 +183,7 @@ def run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: listpages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]pipeline.rpush(pages_key, *[json.dumps(page) for page in pages])pipeline.expire(pages_key, ttl)+ pipeline.expire(counted_urls_key, ttl)pipeline.execute()if new_documents:@@ -111,11 +191,6 @@ def run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: listseed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents))- pipeline = redis.pipeline()- pipeline.hset(record_key, mapping={"status": STATUS_DONE, "finished_at": _now()})- pipeline.expire(record_key, ttl)- pipeline.execute()-def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):"""Crawl breadth first from the seeds, staying within the domains given.@@ -124,22 +199,25 @@ def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):"""frontiers: dict[str, deque[str]] = {}seen_urls = set()+ enqueued_per_domain = Counter()+ max_pages_per_domain = settings.SEED_CRAWL_MAX_PAGES_PER_DOMAINdef enqueue(url: str) -> None:domain = bare_host(url)- if url in seen_urls or domain not in domains:+ if url in seen_urls or domain not in domains or enqueued_per_domain[domain] >= max_pages_per_domain:returnseen_urls.add(url)+ enqueued_per_domain[domain] += 1frontiers.setdefault(domain, deque()).append(url)for url in seed_urls:enqueue(url)- max_pages = settings.SEED_CRAWL_MAX_PAGES- num_crawled = 0+ # The per-domain cap already bounds the rounds; this bounds a round's slow fetches.+ deadline = time.monotonic() + settings.SEED_CRAWL_MAX_SECONDSlast_round_started = None- while num_crawled < max_pages:- batch = [frontier.popleft() for frontier in frontiers.values() if frontier][: max_pages - num_crawled]+ while time.monotonic() < deadline:+ batch = [frontier.popleft() for frontier in frontiers.values() if frontier]if not batch:return@@ -149,7 +227,6 @@ def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):last_round_started = time.monotonic()results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)- num_crawled += len(batch)documents = []for result in results:@@ -174,7 +251,7 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:"""This user's seed crawl for this query without its pages, or None if there is none.Counts the pages rather than reading them, so it stays cheap for a crawl that has added- a thousand.+ thousands."""redis = get_redis_connection("default")record = redis.hgetall(_record_key(user_id, query))diff --git a/mwmbl/main.py b/mwmbl/main.pyindex 659738d..be5797a 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -39,6 +39,26 @@ def run_background_tasks():sleep(TASK_QUEUE_RESTART_SECONDS)+def run_seed_crawls():+ """Run queued seed crawls - see mwmbl.indexer.seed_crawl. Entry point for the child process.++ A process of its own, rather than a task on the queue above, so that crawls lasting+ minutes never hold up the periodic tasks. Spawned for the same reason that is.+ """+ django.setup()++ from mwmbl.indexer.seed_crawl import run_seed_crawl_worker++ while True:+ try:+ run_seed_crawl_worker()+ except Exception:+ # A crawl's own failure is recorded and survived inside the worker, so this is+ # Redis or the database going away.+ logger.exception("Seed crawl worker failed; restarting in %ds", TASK_QUEUE_RESTART_SECONDS)+ sleep(TASK_QUEUE_RESTART_SECONDS)++def run():django.setup()@@ -82,6 +102,12 @@ def run():process.start()logger.info("Started the background task queue (pid %d)", process.pid)+ seed_crawl_process = multiprocessing.get_context("spawn").Process(+ target=run_seed_crawls, name="seed-crawls", daemon=True+ )+ seed_crawl_process.start()+ logger.info("Started the seed crawl worker (pid %d)", seed_crawl_process.pid)+workers = multiprocessing.cpu_count() * 2 + 1class GunicornApp(BaseApplication):diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex c6dc908..c1913e0 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -195,9 +195,9 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = None# Gates database initialisation and background task scheduling; False for the crawler and tests.HAS_DATABASE = True-# Whether this container runs the django-background-tasks queue (see mwmbl.main). Note the-# distinction from HAS_DATABASE above: that gates *scheduling* the tasks, this gates-# *running* them.+# Whether this container runs the django-background-tasks queue and the seed crawl worker+# (see mwmbl.main). Note the distinction from HAS_DATABASE above: that gates *scheduling*+# the tasks, this gates *running* them.## Opt-in rather than on by default because beta shares its database, index and Redis with# production. Exactly one deployment should run the queue, and it should be production:@@ -206,12 +206,6 @@ HAS_DATABASE = True# `dokku config:set <app> RUN_BACKGROUND_TASKS=true`.RUN_BACKGROUND_TASKS = os.environ.get("RUN_BACKGROUND_TASKS", "false").lower() == "true"-# Run tasks on a thread pool rather than one at a time. A seed crawl (below) takes minutes,-# and in series it would hold up the hourly usage sync and the Polar report behind it.-# Each task row is locked while it runs, so no task ever overlaps with itself.-BACKGROUND_TASK_RUN_ASYNC = True-BACKGROUND_TASK_ASYNC_THREADS = 4-SENTRY_DSN = os.environ.get("SENTRY_DSN")@@ -313,7 +307,11 @@ STAAN_TIMEOUT_SECONDS = 5# Seed crawls: Combined Search with crawl=true crawls the Staan results the index lacked,# following links within Staan's domains - see mwmbl.indexer.seed_crawl.-SEED_CRAWL_MAX_PAGES = 1000+SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 100+# A backstop on a crawl's length; the per-domain cap ends almost all of them well before it.+SEED_CRAWL_MAX_SECONDS = 15 * 60+# Crawls waiting for the worker, across all users, beyond which crawl=true queues nothing.+SEED_CRAWL_MAX_QUEUED = 10SEED_CRAWL_THREADS = 8# The least time between two fetches from one domain.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 1.0diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.pyindex 86f9c88..b5772ec 100644--- a/mwmbl/tinysearchengine/combined_search.py+++ b/mwmbl/tinysearchengine/combined_search.py@@ -41,7 +41,6 @@ from ninja_jwt.authentication import JWTAuthfrom pydantic import Fieldfrom mwmbl import pricing-from mwmbl.background import seed_crawlfrom mwmbl.format import format_result_v2from mwmbl.indexer.index_batches import index_results_against_queryfrom mwmbl.indexer.seed_crawl import get_seed_crawl, get_seed_crawl_summary, start_seed_crawl@@ -75,7 +74,8 @@ class CombinedSearchResponse(SearchResponse):)crawl_scheduled: bool = Field(description="Whether this search started a seed crawl (`crawl=true`). False when every "- "EUSP result is already in the index, or when a crawl for this query is still running.",+ "EUSP result is already in the index, when one of your crawls is still queued or running, or "+ "when the crawl queue is full.",examples=[True],)@@ -88,10 +88,15 @@ class NewPage(Schema):class SeedCrawlSummaryResponse(Schema):query: str- status: str = Field(description="`crawling` while the crawl runs, then `done`.", examples=["crawling"])+ status: str = Field(+ description="`queued` until the crawl starts, `crawling` while it runs, then `done`, or `failed`.",+ examples=["crawling"],+ )started_at: strfinished_at: str | None- pages_crawled: int = Field(description="Pages fetched so far, including ones that added nothing.")+ pages_crawled: int = Field(+ description="Pages fetched so far, by every crawl of this query, including ones that added nothing."+ )pages_indexed: int = Field(description="Pages the crawl has added to the Mwmbl index so far.", examples=[42])@@ -123,7 +128,8 @@ DESCRIPTION = ("pages that added.\n\n""With `crawl=true` (JWT only), the search also starts a background crawl of the EUSP results ""that were not already in the Mwmbl index, following their links within the domains EUSP "- f"returned, up to {settings.SEED_CRAWL_MAX_PAGES:,} pages. "+ f"returned, up to {settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN:,} pages per domain. You can have "+ "one crawl queued or running at a time. ""`GET /api/v2/combined-search/new-pages?q=...` lists what it has added, and ""`GET /api/v2/combined-search/new-pages/count?q=...` counts it.\n\n""If EUSP is down or unconfigured, the request loses its extra recall, not its "@@ -245,12 +251,7 @@ def init_router(ranker) -> None:asyncio.to_thread(index_staan_results, q, staan_results),)- crawl_scheduled = False- if crawl:- crawl_args = await sync_to_async(start_seed_crawl)(user.id, q, retrieval.pages, staan_results)- if crawl_args is not None:- await sync_to_async(seed_crawl)(user.id, q, *crawl_args)- crawl_scheduled = True+ crawl_scheduled = crawl and await sync_to_async(start_seed_crawl)(user.id, q, retrieval.pages, staan_results)formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]return CombinedSearchResponse(diff --git a/test/test_combined_search.py b/test/test_combined_search.pyindex d8eb629..acb6ab0 100644--- a/test/test_combined_search.py+++ b/test/test_combined_search.py@@ -8,13 +8,13 @@ provider being down costs recall rather than the request, and that the quota isatomically."""+import jsonimport threadingfrom pathlib import Pathfrom urllib.parse import urlparseimport pytestfrom allauth.account.models import EmailAddress-from background_task.models import Taskfrom django.contrib.auth import get_user_modelfrom django.core.cache import cachefrom django.test import Client, override_settings@@ -529,9 +529,9 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(body = _crawl(client, access_token).json()assert body["crawl_scheduled"] is True- task = Task.objects.get(task_name="mwmbl.background.seed_crawl")- assert task.params() == ([user.id, "tokio", [STAAN_RESULT.url], ["tokio.rs"]], {})- assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"+ job = json.loads(redis_cache.rpop(seed_crawl.QUEUE_KEY))+ assert job == {"user_id": user.id, "query": "tokio", "seed_urls": [STAAN_RESULT.url], "domains": ["tokio.rs"]}+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "queued"@pytest.mark.django_db@@ -539,7 +539,7 @@ def test_no_crawl_without_the_flag(client, access_token, fresh_quota, redis_cachcrawl_sources()assert _get(client, access_token).json()["crawl_scheduled"] is False- assert not Task.objects.filter(task_name="mwmbl.background.seed_crawl").exists()+ assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0@pytest.mark.django_db@@ -549,7 +549,7 @@ def test_no_crawl_when_the_index_already_has_every_staan_result(crawl_sources(index=(INDEX_RESULT, STAAN_RESULT))assert _crawl(client, access_token).json()["crawl_scheduled"] is False- assert not Task.objects.filter(task_name="mwmbl.background.seed_crawl").exists()+ assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0@pytest.mark.django_db@@ -573,7 +573,7 @@ def test_new_pages_reports_the_crawl_for_the_query(client, access_token, fresh_qresponse = client.get(f"{NEW_PAGES_URL}?q=tokio", HTTP_AUTHORIZATION=f"Bearer {access_token}")assert response.status_code == 200- assert response.json()["status"] == "crawling"+ assert response.json()["status"] == "queued"assert response.json()["pages"] == []@@ -613,7 +613,7 @@ def test_new_pages_count_reports_how_many_pages_the_crawl_added(assert response.status_code == 200assert response.json()["pages_indexed"] == 1- assert response.json()["status"] == "crawling"+ assert response.json()["status"] == "queued"assert "pages" not in response.json()diff --git a/test/test_process_tasks_worker.py b/test/test_process_tasks_worker.pyindex 0463744..6509fde 100644--- a/test/test_process_tasks_worker.py+++ b/test/test_process_tasks_worker.py@@ -142,18 +142,17 @@ class _FakeProcess:pass-def test_server_starts_the_queue_when_enabled(settings, fake_server, recorded_commands):+def test_server_starts_the_queue_and_the_seed_crawl_worker_when_enabled(settings, fake_server, recorded_commands):settings.RUN_BACKGROUND_TASKS = Truemain.run()- assert len(fake_server) == 1- started = fake_server[0]- assert started["target"] is main.run_background_tasks- # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis- # sockets. Daemonic so it dies with the container rather than outliving it.- assert started["method"] == "spawn"- assert started["daemon"]+ assert [started["target"] for started in fake_server] == [main.run_background_tasks, main.run_seed_crawls]+ for started in fake_server:+ # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis+ # sockets. Daemonic so it dies with the container rather than outliving it.+ assert started["method"] == "spawn"+ assert started["daemon"]assert FakeGunicorn.runs == 1diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.pyindex 7813788..c662318 100644--- a/test/test_seed_crawl.py+++ b/test/test_seed_crawl.py@@ -3,6 +3,7 @@ and what is recorded for the user. The endpoint that starts a crawl is tested wirest of Combined Search, in test_combined_search.py."""+import jsonfrom pathlib import Pathimport pytest@@ -52,6 +53,13 @@ def fake_web(monkeypatch, settings):return rounds+def worker(monkeypatch):+ """The worker, minus its connection cleanup, which would close the test's transaction."""+ monkeypatch.setattr(seed_crawl, "close_old_connections", lambda: None)+ return seed_crawl.run_next_seed_crawl++@pytest.fixturedef index_path(tmp_path):path = Path(tmp_path) / "index.tinysearch"@@ -98,12 +106,14 @@ def test_each_round_fetches_at_most_one_url_per_domain(fake_web):assert all(len({url.split("/")[2].removeprefix("www.") for url in batch}) == len(batch) for batch in fake_web)-def test_the_crawl_stops_at_the_page_limit(fake_web, settings):- settings.SEED_CRAWL_MAX_PAGES = 2+def test_each_domain_stops_at_its_page_limit(fake_web, settings):+ settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 2- rounds = _crawl([SEED], ["tokio.rs"])+ rounds = _crawl([SEED, "https://rust-lang.github.io/async-book/"], ["tokio.rs", "rust-lang.github.io"])- assert sum(num_crawled for num_crawled, _ in rounds) == 2+ fetched = [url for batch in fake_web for url in batch]+ assert fetched == [SEED, "https://rust-lang.github.io/async-book/", "https://tokio.rs/tutorial"]+ assert sum(num_crawled for num_crawled, _ in rounds) == 3def test_a_failed_fetch_adds_no_document(fake_web):@@ -117,39 +127,57 @@ def test_a_failed_fetch_adds_no_document(fake_web):# ---------------------------------------------------------------------------+def _queued_jobs(redis_cache):+ return [json.loads(job) for job in reversed(redis_cache.lrange(seed_crawl.QUEUE_KEY, 0, -1))]++def test_only_staan_results_missing_from_the_index_are_seeds(redis_cache, monkeypatch):monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())index_pages = [Document("Async book", "https://rust-lang.github.io/async-book/", "")]- seed_urls, domains = seed_crawl.start_seed_crawl(1, "tokio", index_pages, STAAN_RESULTS)+ assert seed_crawl.start_seed_crawl(1, "tokio", index_pages, STAAN_RESULTS) is True- assert seed_urls == [SEED]+ [job] = _queued_jobs(redis_cache)+ assert job["seed_urls"] == [SEED]# Every Staan domain is crawlable, including those of results the index already had.- assert domains == ["rust-lang.github.io", "tokio.rs"]+ assert job["domains"] == ["rust-lang.github.io", "tokio.rs"]def test_blacklisted_staan_results_are_neither_seeds_nor_domains(redis_cache, monkeypatch):monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: {SEED})- seed_urls, domains = seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS)+ seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS)- assert SEED not in seed_urls- assert domains == ["rust-lang.github.io"]+ [job] = _queued_jobs(redis_cache)+ assert SEED not in job["seed_urls"]+ assert job["domains"] == ["rust-lang.github.io"]def test_nothing_to_crawl_starts_nothing(redis_cache, monkeypatch):monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())- assert seed_crawl.start_seed_crawl(1, "tokio", STAAN_RESULTS, STAAN_RESULTS) is None+ assert seed_crawl.start_seed_crawl(1, "tokio", STAAN_RESULTS, STAAN_RESULTS) is Falseassert seed_crawl.get_seed_crawl(1, "tokio") is None+ assert _queued_jobs(redis_cache) == []+++def test_a_user_has_one_crawl_at_a_time(redis_cache, monkeypatch):+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())++ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is True+ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is False+ assert seed_crawl.start_seed_crawl(1, "rust", [], STAAN_RESULTS) is False+ assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is True+ assert len(_queued_jobs(redis_cache)) == 2-def test_a_running_crawl_is_not_started_again(redis_cache, monkeypatch):+def test_nothing_is_queued_once_the_queue_is_full(redis_cache, monkeypatch, settings):monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())+ settings.SEED_CRAWL_MAX_QUEUED = 1- assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is not None- assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is None- assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is not None+ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is True+ assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is False+ assert seed_crawl.get_seed_crawl(2, "tokio") is None# ---------------------------------------------------------------------------@@ -159,6 +187,8 @@ def test_a_running_crawl_is_not_started_again(redis_cache, monkeypatch):def _run(user, index_path, seed_urls=(SEED,)):seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)+ # Crawl only the seeds and domain given, whatever start_seed_crawl made of STAAN_RESULTS.+ seed_crawl.get_redis_connection("default").delete(seed_crawl.QUEUE_KEY)seed_crawl.run_seed_crawl(user.id, "tokio", list(seed_urls), ["tokio.rs"], index_path)return seed_crawl.get_seed_crawl(user.id, "tokio")@@ -183,6 +213,47 @@ def test_the_record_lists_the_new_pages_once_the_crawl_is_done(fake_web, redis_c}+def test_the_worker_runs_a_queued_crawl(fake_web, redis_cache, user, index_path, worker):+ seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)++ assert worker(index_path) is True++ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "done"+ # The user can start another crawl once theirs has finished.+ assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS) is True+++def test_a_failed_crawl_is_recorded_and_not_retried(fake_web, redis_cache, user, index_path, worker, monkeypatch):+ def failing_crawl_batch(urls, num_threads, delay_seconds, redis):+ raise ConnectionError("Redis went away")++ monkeypatch.setattr(seed_crawl, "crawl_batch", failing_crawl_batch)+ seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)++ worker(index_path)++ record = seed_crawl.get_seed_crawl(user.id, "tokio")+ assert record["status"] == "failed"+ assert record["finished_at"] is not None+ assert _queued_jobs(redis_cache) == []+ assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS) is True+++def test_crawling_a_query_again_keeps_its_pages_and_counts_none_twice(fake_web, redis_cache, user, index_path):+ _run(user, index_path)++ record = _run(user, index_path)+ user.refresh_from_db()++ assert record["status"] == "done"+ assert record["pages_indexed"] == 3+ assert record["pages_crawled"] == 6+ assert user.seed_search_pages_indexed == 3++@pytest.mark.django_dbdef test_the_crawled_pages_are_in_the_index(fake_web, redis_cache, user, index_path):_run(user, index_path)