Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Cap seed crawls per domain and run them in a worker of their own

Claude3371bc033cba
diff --git a/mwmbl/background.py b/mwmbl/background.py
index 6d93193..70f313f 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -6,7 +6,6 @@ Django Background Tasks for periodic maintenance:
- report_usage_to_polar: reports billable usage overage to Polar once per hour
- refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against
- purge_blacklisted_from_queue: removes retrieval-filtered documents from the index
- - seed_crawl: crawls outwards from a Combined Search's Staan results, on request
"""
import logging
@@ -28,7 +27,6 @@ from mwmbl.crawler.stats import StatsManager
from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist, refresh_snapshot
from mwmbl.indexer.purge_blacklisted import purge_documents
from mwmbl.indexer.purge_queue import drain_purge_queue, queue_size
-from mwmbl.indexer.seed_crawl import run_seed_crawl
from mwmbl.models import DomainEvidence, DomainSubmission, OldIndex, UsageBucket
from mwmbl.moderation.evidence import crawl_domain, store_evidence, store_failure
from mwmbl.moderation.model import reset_model_cache
@@ -253,16 +251,6 @@ def report_usage_to_polar():
# ---------------------------------------------------------------------------
-@background(schedule=0)
-def seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list[str]):
- """Crawl the pages a Combined Search found that the index lacked - see mwmbl.indexer.seed_crawl.
-
- Takes minutes, which is why the queue runs tasks concurrently (BACKGROUND_TASK_RUN_ASYNC).
- """
- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
- run_seed_crawl(user_id, query, seed_urls, domains, str(index_path))
-
-
@background(schedule=0)
def enrich_domain_submission(domain: str):
"""Crawl a submitted domain and store the suggestion the moderation queue will read.
diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.py
index 8ced6cd..f0ec2e7 100644
--- a/mwmbl/indexer/seed_crawl.py
+++ b/mwmbl/indexer/seed_crawl.py
@@ -1,14 +1,21 @@
"""Seed crawls: crawling outwards from the pages Staan found that the index lacked.
-Combined Search with crawl=true hands the Staan results the index did not already hold to
-a background task (mwmbl.background.seed_crawl), which crawls them and follows their links,
-breadth first, until it has fetched SEED_CRAWL_MAX_PAGES pages. Links are only followed to
-the domains Staan returned for the query, so a crawl stays on the sites the query was about.
+Combined Search with crawl=true queues the Staan results the index did not already hold,
+and a worker process of its own (run_seed_crawl_worker, started by mwmbl.main) crawls them
+and follows their links, breadth first. Links are only followed to the domains Staan
+returned for the query, so a crawl stays on the sites the query was about.
The crawl goes a round at a time, one URL per domain per round and at least
SEED_CRAWL_DOMAIN_DELAY_SECONDS apart, so no site is fetched from faster than that however
-much of the frontier it holds. Each round is indexed as it finishes, so what a crawl has
-added is visible while it is still running.
+much of the frontier it holds. Each domain gives at most SEED_CRAWL_MAX_PAGES_PER_DOMAIN
+pages, which also bounds a crawl to that many rounds - a few minutes - whatever the number
+of domains. Each round is indexed as it finishes, so what a crawl has added is visible while
+it is still running.
+
+The crawls run in their own process rather than on the django-background-tasks queue: they
+take minutes, and in series with the periodic tasks they would hold up the usage sync and
+the Polar report. That process runs them one at a time, each user has at most one queued or
+running, and at most SEED_CRAWL_MAX_QUEUED wait, so nobody can fill the queue.
What it added is recorded in Redis against the user and the query, where
GET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and counted on the user for their stats.
@@ -17,11 +24,13 @@ GET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and count
import hashlib
import json
import time
-from collections import deque
+from collections import Counter, deque
from datetime import datetime, timezone
from logging import getLogger
+from pathlib import Path
from django.conf import settings
+from django.db import close_old_connections
from django.db.models import F
from django_redis import get_redis_connection
@@ -34,8 +43,14 @@ from mwmbl.utils import bare_host
logger = getLogger(__name__)
+STATUS_QUEUED = "queued"
STATUS_CRAWLING = "crawling"
STATUS_DONE = "done"
+STATUS_FAILED = "failed"
+
+QUEUE_KEY = "seed-crawl:queue"
+# Under the Redis client's five-second socket timeout, which a longer blocking pop would trip.
+QUEUE_POLL_SECONDS = 1
def _record_key(user_id: int, query: str) -> str:
@@ -48,54 +63,118 @@ def _pages_key(user_id: int, query: str) -> str:
return f"{_record_key(user_id, query)}:pages"
+def _counted_urls_key(user_id: int, query: str) -> str:
+ return f"{_record_key(user_id, query)}:urls"
+
+
+def _active_key(user_id: int) -> str:
+ return f"seed-crawl:{user_id}:active"
+
+
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
-def start_seed_crawl(user_id: int, query: str, index_pages: list[Document], staan_results: list[Document]):
- """Record a seed crawl as started, returning the seed URLs and domains to crawl it with.
+def start_seed_crawl(user_id: int, query: str, index_pages: list[Document], staan_results: list[Document]) -> bool:
+ """Queue a seed crawl of the Staan results the index lacked, returning whether it did.
- None when there is nothing to crawl - every Staan result is already in the index - or
- when this user's crawl for this query is still running, which a repeated search must
- not start again.
+ Nothing is queued when every Staan result is already in the index, when this user
+ already has a crawl queued or running, or when the queue is full.
"""
blacklisted_urls = find_blacklisted_urls(staan_results)
allowed = [document for document in staan_results if document.url not in blacklisted_urls]
indexed_urls = {document.url for document in index_pages}
seed_urls = [document.url for document in allowed if document.url not in indexed_urls]
if not seed_urls:
- return None
+ return False
redis = get_redis_connection("default")
- record_key = _record_key(user_id, query)
- if redis.hget(record_key, "status") == STATUS_CRAWLING.encode():
- return None
+ # A slight overshoot from racing requests is harmless; the cap is there to stop one
+ # user, or many, queueing hours of crawling.
+ if redis.llen(QUEUE_KEY) >= settings.SEED_CRAWL_MAX_QUEUED:
+ return False
+ # Set atomically, so a double click or a retried request cannot queue the crawl twice.
+ # The expiry outlasts the longest wait plus the crawl, and frees the user should the
+ # worker die mid-crawl.
+ lock_seconds = settings.SEED_CRAWL_MAX_SECONDS * (settings.SEED_CRAWL_MAX_QUEUED + 1)
+ if not redis.set(_active_key(user_id), query, nx=True, ex=lock_seconds):
+ return False
- ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS
+ record_key = _record_key(user_id, query)
+ domains = sorted({bare_host(document.url) for document in allowed})
+ job = {"user_id": user_id, "query": query, "seed_urls": seed_urls, "domains": domains}
+ # An earlier crawl's pages are kept: the new one adds to them, and the user's total
+ # already includes them.
pipeline = redis.pipeline()
- pipeline.delete(record_key, _pages_key(user_id, query))
- pipeline.hset(
- record_key,
- mapping={"status": STATUS_CRAWLING, "query": query, "started_at": _now(), "pages_crawled": 0},
- )
- pipeline.expire(record_key, ttl)
+ pipeline.hset(record_key, mapping={"status": STATUS_QUEUED, "query": query, "started_at": _now()})
+ pipeline.hsetnx(record_key, "pages_crawled", 0)
+ pipeline.hdel(record_key, "finished_at")
+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)
+ pipeline.lpush(QUEUE_KEY, json.dumps(job))
pipeline.execute()
+ return True
- domains = sorted({bare_host(document.url) for document in allowed})
- return seed_urls, domains
+
+def run_seed_crawl_worker() -> None:
+ """Run queued seed crawls, one at a time, for ever."""
+ index_path = str(Path(settings.DATA_PATH) / settings.INDEX_NAME)
+ while True:
+ run_next_seed_crawl(index_path)
+
+
+def run_next_seed_crawl(index_path: str) -> bool:
+ """Run the next queued seed crawl, if one arrives within QUEUE_POLL_SECONDS."""
+ redis = get_redis_connection("default")
+ popped = redis.brpop(QUEUE_KEY, timeout=QUEUE_POLL_SECONDS)
+ if popped is None:
+ return False
+ # A worker that waits for hours between crawls would otherwise reuse a connection
+ # Postgres has long since closed.
+ close_old_connections()
+ job = json.loads(popped[1])
+ run_seed_crawl(job["user_id"], job["query"], job["seed_urls"], job["domains"], index_path)
+ return True
def run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list[str], index_path: str) -> None:
redis = get_redis_connection("default")
+ record_key = _record_key(user_id, query)
+ redis.hset(record_key, "status", STATUS_CRAWLING)
+ try:
+ _crawl_and_record(user_id, query, seed_urls, domains, index_path, redis)
+ status = STATUS_DONE
+ except Exception:
+ # Recorded rather than retried: a retry would start again from the seeds, fetching
+ # every site a second time. The worker carries on with the next crawl.
+ logger.exception("Seed crawl for user %d failed", user_id)
+ status = STATUS_FAILED
+ finally:
+ redis.delete(_active_key(user_id))
+
+ pipeline = redis.pipeline()
+ pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})
+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)
+ pipeline.execute()
+
+
+def _crawl_and_record(user_id: int, query: str, seed_urls: list[str], domains: list[str], index_path: str, redis):
record_key = _record_key(user_id, query)
pages_key = _pages_key(user_id, query)
+ counted_urls_key = _counted_urls_key(user_id, query)
ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS
for num_crawled, documents in crawl_within_domains(seed_urls, set(domains), redis):
# The seeds were missing from the index when the user searched, but Combined Search
# has since written Staan's snippets of them, so the index alone no longer says so.
new_urls = index_new_documents(documents, index_path) | set(seed_urls)
- new_documents = [document for document in documents if document.url in new_urls]
+ candidates = [document for document in documents if document.url in new_urls]
+
+ # A page is counted once per query, however many crawls of it find it new.
+ pipeline = redis.pipeline()
+ for document in candidates:
+ pipeline.sadd(counted_urls_key, document.url)
+ first_counted = pipeline.execute()
+ new_documents = [document for document, added in zip(candidates, first_counted) if added]
logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))
pipeline = redis.pipeline()
@@ -104,6 +183,7 @@ def run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list
pages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]
pipeline.rpush(pages_key, *[json.dumps(page) for page in pages])
pipeline.expire(pages_key, ttl)
+ pipeline.expire(counted_urls_key, ttl)
pipeline.execute()
if new_documents:
@@ -111,11 +191,6 @@ def run_seed_crawl(user_id: int, query: str, seed_urls: list[str], domains: list
seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)
)
- pipeline = redis.pipeline()
- pipeline.hset(record_key, mapping={"status": STATUS_DONE, "finished_at": _now()})
- pipeline.expire(record_key, ttl)
- pipeline.execute()
-
def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):
"""Crawl breadth first from the seeds, staying within the domains given.
@@ -124,22 +199,25 @@ def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):
"""
frontiers: dict[str, deque[str]] = {}
seen_urls = set()
+ enqueued_per_domain = Counter()
+ max_pages_per_domain = settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN
def enqueue(url: str) -> None:
domain = bare_host(url)
- if url in seen_urls or domain not in domains:
+ if url in seen_urls or domain not in domains or enqueued_per_domain[domain] >= max_pages_per_domain:
return
seen_urls.add(url)
+ enqueued_per_domain[domain] += 1
frontiers.setdefault(domain, deque()).append(url)
for url in seed_urls:
enqueue(url)
- max_pages = settings.SEED_CRAWL_MAX_PAGES
- num_crawled = 0
+ # The per-domain cap already bounds the rounds; this bounds a round's slow fetches.
+ deadline = time.monotonic() + settings.SEED_CRAWL_MAX_SECONDS
last_round_started = None
- while num_crawled < max_pages:
- batch = [frontier.popleft() for frontier in frontiers.values() if frontier][: max_pages - num_crawled]
+ while time.monotonic() < deadline:
+ batch = [frontier.popleft() for frontier in frontiers.values() if frontier]
if not batch:
return
@@ -149,7 +227,6 @@ def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):
last_round_started = time.monotonic()
results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)
- num_crawled += len(batch)
documents = []
for result in results:
@@ -174,7 +251,7 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:
"""This user's seed crawl for this query without its pages, or None if there is none.
Counts the pages rather than reading them, so it stays cheap for a crawl that has added
- a thousand.
+ thousands.
"""
redis = get_redis_connection("default")
record = redis.hgetall(_record_key(user_id, query))
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 659738d..be5797a 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -39,6 +39,26 @@ def run_background_tasks():
sleep(TASK_QUEUE_RESTART_SECONDS)
+def run_seed_crawls():
+ """Run queued seed crawls - see mwmbl.indexer.seed_crawl. Entry point for the child process.
+
+ A process of its own, rather than a task on the queue above, so that crawls lasting
+ minutes never hold up the periodic tasks. Spawned for the same reason that is.
+ """
+ django.setup()
+
+ from mwmbl.indexer.seed_crawl import run_seed_crawl_worker
+
+ while True:
+ try:
+ run_seed_crawl_worker()
+ except Exception:
+ # A crawl's own failure is recorded and survived inside the worker, so this is
+ # Redis or the database going away.
+ logger.exception("Seed crawl worker failed; restarting in %ds", TASK_QUEUE_RESTART_SECONDS)
+ sleep(TASK_QUEUE_RESTART_SECONDS)
+
+
def run():
django.setup()
@@ -82,6 +102,12 @@ def run():
process.start()
logger.info("Started the background task queue (pid %d)", process.pid)
+ seed_crawl_process = multiprocessing.get_context("spawn").Process(
+ target=run_seed_crawls, name="seed-crawls", daemon=True
+ )
+ seed_crawl_process.start()
+ logger.info("Started the seed crawl worker (pid %d)", seed_crawl_process.pid)
+
workers = multiprocessing.cpu_count() * 2 + 1
class GunicornApp(BaseApplication):
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index c6dc908..c1913e0 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -195,9 +195,9 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = None
# Gates database initialisation and background task scheduling; False for the crawler and tests.
HAS_DATABASE = True
-# Whether this container runs the django-background-tasks queue (see mwmbl.main). Note the
-# distinction from HAS_DATABASE above: that gates *scheduling* the tasks, this gates
-# *running* them.
+# Whether this container runs the django-background-tasks queue and the seed crawl worker
+# (see mwmbl.main). Note the distinction from HAS_DATABASE above: that gates *scheduling*
+# the tasks, this gates *running* them.
#
# Opt-in rather than on by default because beta shares its database, index and Redis with
# production. Exactly one deployment should run the queue, and it should be production:
@@ -206,12 +206,6 @@ HAS_DATABASE = True
# `dokku config:set <app> RUN_BACKGROUND_TASKS=true`.
RUN_BACKGROUND_TASKS = os.environ.get("RUN_BACKGROUND_TASKS", "false").lower() == "true"
-# Run tasks on a thread pool rather than one at a time. A seed crawl (below) takes minutes,
-# and in series it would hold up the hourly usage sync and the Polar report behind it.
-# Each task row is locked while it runs, so no task ever overlaps with itself.
-BACKGROUND_TASK_RUN_ASYNC = True
-BACKGROUND_TASK_ASYNC_THREADS = 4
-
SENTRY_DSN = os.environ.get("SENTRY_DSN")
@@ -313,7 +307,11 @@ STAAN_TIMEOUT_SECONDS = 5
# Seed crawls: Combined Search with crawl=true crawls the Staan results the index lacked,
# following links within Staan's domains - see mwmbl.indexer.seed_crawl.
-SEED_CRAWL_MAX_PAGES = 1000
+SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 100
+# A backstop on a crawl's length; the per-domain cap ends almost all of them well before it.
+SEED_CRAWL_MAX_SECONDS = 15 * 60
+# Crawls waiting for the worker, across all users, beyond which crawl=true queues nothing.
+SEED_CRAWL_MAX_QUEUED = 10
SEED_CRAWL_THREADS = 8
# The least time between two fetches from one domain.
SEED_CRAWL_DOMAIN_DELAY_SECONDS = 1.0
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.py
index 86f9c88..b5772ec 100644
--- a/mwmbl/tinysearchengine/combined_search.py
+++ b/mwmbl/tinysearchengine/combined_search.py
@@ -41,7 +41,6 @@ from ninja_jwt.authentication import JWTAuth
from pydantic import Field
from mwmbl import pricing
-from mwmbl.background import seed_crawl
from mwmbl.format import format_result_v2
from mwmbl.indexer.index_batches import index_results_against_query
from mwmbl.indexer.seed_crawl import get_seed_crawl, get_seed_crawl_summary, start_seed_crawl
@@ -75,7 +74,8 @@ class CombinedSearchResponse(SearchResponse):
)
crawl_scheduled: bool = Field(
description="Whether this search started a seed crawl (`crawl=true`). False when every "
- "EUSP result is already in the index, or when a crawl for this query is still running.",
+ "EUSP result is already in the index, when one of your crawls is still queued or running, or "
+ "when the crawl queue is full.",
examples=[True],
)
@@ -88,10 +88,15 @@ class NewPage(Schema):
class SeedCrawlSummaryResponse(Schema):
query: str
- status: str = Field(description="`crawling` while the crawl runs, then `done`.", examples=["crawling"])
+ status: str = Field(
+ description="`queued` until the crawl starts, `crawling` while it runs, then `done`, or `failed`.",
+ examples=["crawling"],
+ )
started_at: str
finished_at: str | None
- pages_crawled: int = Field(description="Pages fetched so far, including ones that added nothing.")
+ pages_crawled: int = Field(
+ description="Pages fetched so far, by every crawl of this query, including ones that added nothing."
+ )
pages_indexed: int = Field(description="Pages the crawl has added to the Mwmbl index so far.", examples=[42])
@@ -123,7 +128,8 @@ DESCRIPTION = (
"pages that added.\n\n"
"With `crawl=true` (JWT only), the search also starts a background crawl of the EUSP results "
"that were not already in the Mwmbl index, following their links within the domains EUSP "
- f"returned, up to {settings.SEED_CRAWL_MAX_PAGES:,} pages. "
+ f"returned, up to {settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN:,} pages per domain. You can have "
+ "one crawl queued or running at a time. "
"`GET /api/v2/combined-search/new-pages?q=...` lists what it has added, and "
"`GET /api/v2/combined-search/new-pages/count?q=...` counts it.\n\n"
"If EUSP is down or unconfigured, the request loses its extra recall, not its "
@@ -245,12 +251,7 @@ def init_router(ranker) -> None:
asyncio.to_thread(index_staan_results, q, staan_results),
)
- crawl_scheduled = False
- if crawl:
- crawl_args = await sync_to_async(start_seed_crawl)(user.id, q, retrieval.pages, staan_results)
- if crawl_args is not None:
- await sync_to_async(seed_crawl)(user.id, q, *crawl_args)
- crawl_scheduled = True
+ crawl_scheduled = crawl and await sync_to_async(start_seed_crawl)(user.id, q, retrieval.pages, staan_results)
formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]
return CombinedSearchResponse(
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
index d8eb629..acb6ab0 100644
--- a/test/test_combined_search.py
+++ b/test/test_combined_search.py
@@ -8,13 +8,13 @@ provider being down costs recall rather than the request, and that the quota is
atomically.
"""
+import json
import threading
from pathlib import Path
from urllib.parse import urlparse
import pytest
from allauth.account.models import EmailAddress
-from background_task.models import Task
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.test import Client, override_settings
@@ -529,9 +529,9 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(
body = _crawl(client, access_token).json()
assert body["crawl_scheduled"] is True
- task = Task.objects.get(task_name="mwmbl.background.seed_crawl")
- assert task.params() == ([user.id, "tokio", [STAAN_RESULT.url], ["tokio.rs"]], {})
- assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"
+ job = json.loads(redis_cache.rpop(seed_crawl.QUEUE_KEY))
+ assert job == {"user_id": user.id, "query": "tokio", "seed_urls": [STAAN_RESULT.url], "domains": ["tokio.rs"]}
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "queued"
@pytest.mark.django_db
@@ -539,7 +539,7 @@ def test_no_crawl_without_the_flag(client, access_token, fresh_quota, redis_cach
crawl_sources()
assert _get(client, access_token).json()["crawl_scheduled"] is False
- assert not Task.objects.filter(task_name="mwmbl.background.seed_crawl").exists()
+ assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0
@pytest.mark.django_db
@@ -549,7 +549,7 @@ def test_no_crawl_when_the_index_already_has_every_staan_result(
crawl_sources(index=(INDEX_RESULT, STAAN_RESULT))
assert _crawl(client, access_token).json()["crawl_scheduled"] is False
- assert not Task.objects.filter(task_name="mwmbl.background.seed_crawl").exists()
+ assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0
@pytest.mark.django_db
@@ -573,7 +573,7 @@ def test_new_pages_reports_the_crawl_for_the_query(client, access_token, fresh_q
response = client.get(f"{NEW_PAGES_URL}?q=tokio", HTTP_AUTHORIZATION=f"Bearer {access_token}")
assert response.status_code == 200
- assert response.json()["status"] == "crawling"
+ assert response.json()["status"] == "queued"
assert response.json()["pages"] == []
@@ -613,7 +613,7 @@ def test_new_pages_count_reports_how_many_pages_the_crawl_added(
assert response.status_code == 200
assert response.json()["pages_indexed"] == 1
- assert response.json()["status"] == "crawling"
+ assert response.json()["status"] == "queued"
assert "pages" not in response.json()
diff --git a/test/test_process_tasks_worker.py b/test/test_process_tasks_worker.py
index 0463744..6509fde 100644
--- a/test/test_process_tasks_worker.py
+++ b/test/test_process_tasks_worker.py
@@ -142,18 +142,17 @@ class _FakeProcess:
pass
-def test_server_starts_the_queue_when_enabled(settings, fake_server, recorded_commands):
+def test_server_starts_the_queue_and_the_seed_crawl_worker_when_enabled(settings, fake_server, recorded_commands):
settings.RUN_BACKGROUND_TASKS = True
main.run()
- assert len(fake_server) == 1
- started = fake_server[0]
- assert started["target"] is main.run_background_tasks
- # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis
- # sockets. Daemonic so it dies with the container rather than outliving it.
- assert started["method"] == "spawn"
- assert started["daemon"]
+ assert [started["target"] for started in fake_server] == [main.run_background_tasks, main.run_seed_crawls]
+ for started in fake_server:
+ # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis
+ # sockets. Daemonic so it dies with the container rather than outliving it.
+ assert started["method"] == "spawn"
+ assert started["daemon"]
assert FakeGunicorn.runs == 1
diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.py
index 7813788..c662318 100644
--- a/test/test_seed_crawl.py
+++ b/test/test_seed_crawl.py
@@ -3,6 +3,7 @@ and what is recorded for the user. The endpoint that starts a crawl is tested wi
rest of Combined Search, in test_combined_search.py.
"""
+import json
from pathlib import Path
import pytest
@@ -52,6 +53,13 @@ def fake_web(monkeypatch, settings):
return rounds
+def worker(monkeypatch):
+ """The worker, minus its connection cleanup, which would close the test's transaction."""
+ monkeypatch.setattr(seed_crawl, "close_old_connections", lambda: None)
+ return seed_crawl.run_next_seed_crawl
+
+
@pytest.fixture
def index_path(tmp_path):
path = Path(tmp_path) / "index.tinysearch"
@@ -98,12 +106,14 @@ def test_each_round_fetches_at_most_one_url_per_domain(fake_web):
assert all(len({url.split("/")[2].removeprefix("www.") for url in batch}) == len(batch) for batch in fake_web)
-def test_the_crawl_stops_at_the_page_limit(fake_web, settings):
- settings.SEED_CRAWL_MAX_PAGES = 2
+def test_each_domain_stops_at_its_page_limit(fake_web, settings):
+ settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 2
- rounds = _crawl([SEED], ["tokio.rs"])
+ rounds = _crawl([SEED, "https://rust-lang.github.io/async-book/"], ["tokio.rs", "rust-lang.github.io"])
- assert sum(num_crawled for num_crawled, _ in rounds) == 2
+ fetched = [url for batch in fake_web for url in batch]
+ assert fetched == [SEED, "https://rust-lang.github.io/async-book/", "https://tokio.rs/tutorial"]
+ assert sum(num_crawled for num_crawled, _ in rounds) == 3
def test_a_failed_fetch_adds_no_document(fake_web):
@@ -117,39 +127,57 @@ def test_a_failed_fetch_adds_no_document(fake_web):
# ---------------------------------------------------------------------------
+def _queued_jobs(redis_cache):
+ return [json.loads(job) for job in reversed(redis_cache.lrange(seed_crawl.QUEUE_KEY, 0, -1))]
+
+
def test_only_staan_results_missing_from_the_index_are_seeds(redis_cache, monkeypatch):
monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
index_pages = [Document("Async book", "https://rust-lang.github.io/async-book/", "")]
- seed_urls, domains = seed_crawl.start_seed_crawl(1, "tokio", index_pages, STAAN_RESULTS)
+ assert seed_crawl.start_seed_crawl(1, "tokio", index_pages, STAAN_RESULTS) is True
- assert seed_urls == [SEED]
+ [job] = _queued_jobs(redis_cache)
+ assert job["seed_urls"] == [SEED]
# Every Staan domain is crawlable, including those of results the index already had.
- assert domains == ["rust-lang.github.io", "tokio.rs"]
+ assert job["domains"] == ["rust-lang.github.io", "tokio.rs"]
def test_blacklisted_staan_results_are_neither_seeds_nor_domains(redis_cache, monkeypatch):
monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: {SEED})
- seed_urls, domains = seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS)
+ seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS)
- assert SEED not in seed_urls
- assert domains == ["rust-lang.github.io"]
+ [job] = _queued_jobs(redis_cache)
+ assert SEED not in job["seed_urls"]
+ assert job["domains"] == ["rust-lang.github.io"]
def test_nothing_to_crawl_starts_nothing(redis_cache, monkeypatch):
monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
- assert seed_crawl.start_seed_crawl(1, "tokio", STAAN_RESULTS, STAAN_RESULTS) is None
+ assert seed_crawl.start_seed_crawl(1, "tokio", STAAN_RESULTS, STAAN_RESULTS) is False
assert seed_crawl.get_seed_crawl(1, "tokio") is None
+ assert _queued_jobs(redis_cache) == []
+
+
+def test_a_user_has_one_crawl_at_a_time(redis_cache, monkeypatch):
+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
+
+ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is True
+ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is False
+ assert seed_crawl.start_seed_crawl(1, "rust", [], STAAN_RESULTS) is False
+ assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is True
+ assert len(_queued_jobs(redis_cache)) == 2
-def test_a_running_crawl_is_not_started_again(redis_cache, monkeypatch):
+def test_nothing_is_queued_once_the_queue_is_full(redis_cache, monkeypatch, settings):
monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
+ settings.SEED_CRAWL_MAX_QUEUED = 1
- assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is not None
- assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is None
- assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is not None
+ assert seed_crawl.start_seed_crawl(1, "tokio", [], STAAN_RESULTS) is True
+ assert seed_crawl.start_seed_crawl(2, "tokio", [], STAAN_RESULTS) is False
+ assert seed_crawl.get_seed_crawl(2, "tokio") is None
# ---------------------------------------------------------------------------
@@ -159,6 +187,8 @@ def test_a_running_crawl_is_not_started_again(redis_cache, monkeypatch):
def _run(user, index_path, seed_urls=(SEED,)):
seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)
+ # Crawl only the seeds and domain given, whatever start_seed_crawl made of STAAN_RESULTS.
+ seed_crawl.get_redis_connection("default").delete(seed_crawl.QUEUE_KEY)
seed_crawl.run_seed_crawl(user.id, "tokio", list(seed_urls), ["tokio.rs"], index_path)
return seed_crawl.get_seed_crawl(user.id, "tokio")
@@ -183,6 +213,47 @@ def test_the_record_lists_the_new_pages_once_the_crawl_is_done(fake_web, redis_c
}
+def test_the_worker_runs_a_queued_crawl(fake_web, redis_cache, user, index_path, worker):
+ seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)
+
+ assert worker(index_path) is True
+
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "done"
+ # The user can start another crawl once theirs has finished.
+ assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS) is True
+
+
+def test_a_failed_crawl_is_recorded_and_not_retried(fake_web, redis_cache, user, index_path, worker, monkeypatch):
+ def failing_crawl_batch(urls, num_threads, delay_seconds, redis):
+ raise ConnectionError("Redis went away")
+
+ monkeypatch.setattr(seed_crawl, "crawl_batch", failing_crawl_batch)
+ seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS)
+
+ worker(index_path)
+
+ record = seed_crawl.get_seed_crawl(user.id, "tokio")
+ assert record["status"] == "failed"
+ assert record["finished_at"] is not None
+ assert _queued_jobs(redis_cache) == []
+ assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS) is True
+
+
+def test_crawling_a_query_again_keeps_its_pages_and_counts_none_twice(fake_web, redis_cache, user, index_path):
+ _run(user, index_path)
+
+ record = _run(user, index_path)
+ user.refresh_from_db()
+
+ assert record["status"] == "done"
+ assert record["pages_indexed"] == 3
+ assert record["pages_crawled"] == 6
+ assert user.seed_search_pages_indexed == 3
+
+
@pytest.mark.django_db
def test_the_crawled_pages_are_in_the_index(fake_web, redis_cache, user, index_path):
_run(user, index_path)