An open source, non-profit web search engine
Run seed crawls as leased, resumable rounds
Daoud Clarkec61847feab81
diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.pyindex f2073a8..f7aca8f 100644--- a/mwmbl/indexer/seed_crawl.py+++ b/mwmbl/indexer/seed_crawl.py@@ -12,21 +12,34 @@ pages, which also bounds a crawl to that many rounds - a few minutes - whateverof domains. Each round is indexed as it finishes, so what a crawl has added is visible whileit is still running.+Everything about a crawl is in one Redis hash, its record, which is read by+GET /api/v2/combined-search/new-pages (and /new-pages/count). That includes its frontier,+so a crawl is a series of rounds that any worker can pick up, rather than minutes of state+held in one process. A crawl queued or running is in READY_KEY, scored by when it was+queued, so the oldest is crawled to the end before the next starts. A worker claims it by+pushing its score out by SEED_CRAWL_LEASE_SECONDS, runs one round, and writes the round's+outcome together with the new frontier and the crawl's own score back - or, at the end, the+crawl's status - in one transaction. A worker+that dies mid-round never writes, so once its lease runs out the crawl is claimed again+from the round it was on; one that is merely slow is shut out by the claim's token, which+the next claim changes. So there is nothing to clean up after a worker, however it stopped.+The crawls run in their own process rather than on the django-background-tasks queue: theytake minutes, and in series with the periodic tasks they would hold up the usage sync and-the Polar report. That process runs them one at a time, each user has at most one queued or-running, and at most SEED_CRAWL_MAX_QUEUED wait, so nobody can fill the queue.+the Polar report. One crawl runs at a time, each user has at most one+queued or running, and at most SEED_CRAWL_MAX_QUEUED are, so nobody can fill the queue.-What it added is recorded in Redis against the user and the query, where-GET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and counted on the user for their stats.-That record also says how far along the crawl is, and what it found on each of Staan's-domains, which are scored for crawlers as seed domains - see mwmbl.indexer.seed_domains.+The new pages a crawl added are counted on the user for their stats, and what it found on+each of Staan's domains is scored for crawlers as seed domains - see+mwmbl.indexer.seed_domains."""import hashlibimport jsonimport time-from collections import Counter, deque+from collections import Counter+from collections.abc import Callable+from dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom enum import StrEnumfrom http import HTTPStatus@@ -38,6 +51,8 @@ from django.conf import settingsfrom django.db import close_old_connectionsfrom django.db.models import Ffrom django_redis import get_redis_connection+from redis import WatchError+from redis.client import Pipelinefrom mwmbl.crawler.retrieve import crawl_batchfrom mwmbl.indexer.index_batches import index_new_documents@@ -66,9 +81,11 @@ class CrawlOutcome(StrEnum):QUEUE_FULL = "queue_full"-QUEUE_KEY = "seed-crawl:queue"-# Under the Redis client's five-second socket timeout, which a longer blocking pop would trip.-QUEUE_POLL_SECONDS = 1+# The records of the crawls queued or running, scored by when each was queued, or while a+# worker holds one, by when its lease ends.+READY_KEY = "seed-crawl:ready"+# How long the worker waits before looking again when no crawl is due.+POLL_SECONDS = 1def _record_key(user_id: int, query: str) -> str:@@ -77,26 +94,130 @@ def _record_key(user_id: int, query: str) -> str:return f"seed-crawl:{user_id}:{query_hash}"-def _pages_key(user_id: int, query: str) -> str:- return f"{_record_key(user_id, query)}:pages"+def _pages_key(record_key: str) -> str:+ return f"{record_key}:pages"-def _counted_urls_key(user_id: int, query: str) -> str:- return f"{_record_key(user_id, query)}:urls"+def _counted_urls_key(record_key: str) -> str:+ return f"{record_key}:urls"-def _domain_pages_key(user_id: int, query: str) -> str:- return f"{_record_key(user_id, query)}:domain-pages"+def _domain_pages_key(record_key: str) -> str:+ return f"{record_key}:domain-pages"def _active_key(user_id: int) -> str:- return f"seed-crawl:{user_id}:active"+ # Named apart from the expiring lock that held the user's query before crawls were+ # leased, so those left behind by the deploy cannot block anyone.+ return f"seed-crawl:{user_id}:active-crawl"def _now() -> str:return datetime.now(timezone.utc).isoformat()+@dataclass+class Frontier:+ """What is left of a crawl: the URLs still to fetch from each domain, and its counts.++ A domain is dropped once SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN fetches in a row from it+ have added nothing new, so a crawl ends when there is nothing left for it to find rather+ than when it has fetched every domain's budget.+ """++ domains: set[str]+ queues: dict[str, list[str]] = field(default_factory=dict)+ seen_urls: set[str] = field(default_factory=set)+ enqueued_per_domain: Counter = field(default_factory=Counter)+ stale_fetches_per_domain: Counter = field(default_factory=Counter)++ @classmethod+ def from_seeds(cls, seed_urls: list[str], domains: set[str]) -> "Frontier":+ frontier = cls(domains)+ for url in seed_urls:+ frontier.enqueue(url)+ return frontier++ @classmethod+ def from_json(cls, data: str) -> "Frontier":+ fields = json.loads(data)+ return cls(+ set(fields["domains"]),+ fields["queues"],+ set(fields["seen_urls"]),+ Counter(fields["enqueued_per_domain"]),+ Counter(fields["stale_fetches_per_domain"]),+ )++ def to_json(self) -> str:+ return json.dumps(+ {+ "domains": sorted(self.domains),+ "queues": self.queues,+ "seen_urls": sorted(self.seen_urls),+ "enqueued_per_domain": self.enqueued_per_domain,+ "stale_fetches_per_domain": self.stale_fetches_per_domain,+ }+ )++ def enqueue(self, url: str) -> None:+ domain = bare_host(url)+ if (+ url in self.seen_urls+ or domain not in self.domains+ or self.enqueued_per_domain[domain] >= settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN+ ):+ return+ self.seen_urls.add(url)+ self.enqueued_per_domain[domain] += 1+ self.queues.setdefault(domain, []).append(url)++ def follow(self, link: str) -> None:+ # Query strings are where faceted search, sorting and calendars multiply one page+ # into thousands, which would spend a domain's budget on near-duplicates.+ if urlsplit(link).query:+ return+ self.enqueue(link)++ def next_batch(self) -> list[str]:+ """Take the next URL of each domain that has one."""+ return [queue.pop(0) for queue in self.queues.values() if queue]++ def record_round(self, batch: list[str], new_urls: set[str]) -> None:+ """Note which of a round's URLs gave new pages, dropping the domains that have run dry."""+ for url in batch:+ domain = bare_host(url)+ if url in new_urls:+ self.stale_fetches_per_domain[domain] = 0+ continue+ self.stale_fetches_per_domain[domain] += 1+ if self.stale_fetches_per_domain[domain] >= settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN:+ self.queues[domain].clear()+++def crawl_round(frontier: Frontier, batch: list[str], redis) -> list[Document]:+ """Fetch a round's URLs, adding their links to the frontier and returning their documents."""+ results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)+ documents = []+ for result in results:+ content = result["content"]+ # An error page is not content, and its links are no lead to any.+ if content is None or result["status"] != HTTPStatus.OK:+ continue+ if content["title"]:+ documents.append(+ Document(+ content["title"],+ result["url"],+ content["extract"],+ last_crawled=result["timestamp"] // 1000,+ )+ )+ for link in content["links"] + content["extra_links"]:+ frontier.follow(link)+ return documents++def start_seed_crawl(user_id: int, query: str, index_pages: list[Document], staan_results: list[Document], new_staan_urls: set[str]) -> CrawlOutcome:@@ -119,249 +240,275 @@ def start_seed_crawl(if not seed_urls:return CrawlOutcome.ALREADY_INDEXED- redis = get_redis_connection("default")- record_key = _record_key(user_id, query)- # Pages a crawl adds need not rank for the query, so its seeds would mostly be the same- # again: a query is crawled once while its record lasts, unless that crawl failed.- status = redis.hget(record_key, "status")- if status in (STATUS_QUEUED.encode(), STATUS_CRAWLING.encode()):- return CrawlOutcome.ALREADY_RUNNING- if status == STATUS_DONE.encode():- return CrawlOutcome.ALREADY_CRAWLED- # A slight overshoot from racing requests is harmless; the cap is there to stop one- # user, or many, queueing hours of crawling.- if redis.llen(QUEUE_KEY) >= settings.SEED_CRAWL_MAX_QUEUED:- return CrawlOutcome.QUEUE_FULL- # Set atomically, so a double click or a retried request cannot queue the crawl twice.- # The expiry outlasts the longest wait plus the crawl, and frees the user should the- # worker die mid-crawl.- lock_seconds = settings.SEED_CRAWL_MAX_SECONDS * (settings.SEED_CRAWL_MAX_QUEUED + 1)- if not redis.set(_active_key(user_id), query, nx=True, ex=lock_seconds):- return CrawlOutcome.ALREADY_RUNNING-# Each domain with the number of Staan results it had, which the seed domains count.staan_results_per_domain = Counter(bare_host(document.url) for document in allowed)domains = dict(sorted(staan_results_per_domain.items()))new_seed_urls = [url for url in seed_urls if url in new_staan_urls]- job = {- "user_id": user_id,- "query": query,- "seed_urls": seed_urls,- "new_seed_urls": new_seed_urls,- "domains": domains,- }- # A failed earlier crawl's pages are kept: the new one adds to them, and the user's total- # already includes them. Its progress and domains are this crawl's alone.+ return queue_seed_crawl(user_id, query, seed_urls, new_seed_urls, domains)+++# Queues a crawl, unless the user already has one queued or running, has crawled the query+# already, or the queue is full. One script, so the user's active crawl, the ready set and+# the record can never disagree. A failed earlier crawl's pages are kept: the new one adds+# to them, and the user's total already includes them. Its progress and domains are this+# crawl's alone. The token is bumped so that no worker still holding an earlier crawl of the+# query can write to this one.+_QUEUE_SCRIPT = """+local record, active, ready, domain_pages = KEYS[1], KEYS[2], KEYS[3], KEYS[4]+if redis.call('ZSCORE', ready, record) then+ return 'already_running'+end+if redis.call('HGET', record, 'status') == 'done' then+ return 'already_crawled'+end+if redis.call('EXISTS', active) == 1 then+ return 'already_running'+end+if redis.call('ZCARD', ready) >= tonumber(ARGV[1]) then+ return 'queue_full'+end+redis.call('HSET', record, unpack(ARGV, 4))+redis.call('HSETNX', record, 'pages_crawled', 0)+redis.call('HDEL', record, 'finished_at', 'crawl_started_at', 'discovered', 'attempts', 'last_round_started')+redis.call('HINCRBY', record, 'token', 1)+redis.call('EXPIRE', record, ARGV[3])+redis.call('DEL', domain_pages)+redis.call('SET', active, record)+redis.call('ZADD', ready, ARGV[2], record)+return 'scheduled'+"""+++def queue_seed_crawl(+ user_id: int, query: str, seed_urls: list[str], new_seed_urls: list[str], domains: dict[str, int]+) -> CrawlOutcome:+ """Queue a crawl from these seeds within these domains, unless the user may not have one."""+ record_key = _record_key(user_id, query)+ queued_at = time.time()record = {"status": STATUS_QUEUED,+ "user_id": user_id,"query": query,"started_at": _now(),+ "queued_at": queued_at,"rounds": 0,"domains": json.dumps(domains),+ "new_seed_urls": json.dumps(new_seed_urls),+ "frontier": Frontier.from_seeds(seed_urls, set(domains)).to_json(),}- pipeline = redis.pipeline()- pipeline.hset(record_key, mapping=record)- pipeline.hsetnx(record_key, "pages_crawled", 0)- pipeline.hdel(record_key, "finished_at", "crawl_started_at", "discovered")- pipeline.delete(_domain_pages_key(user_id, query))- pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)- pipeline.lpush(QUEUE_KEY, json.dumps(job))- pipeline.execute()- return CrawlOutcome.SCHEDULED+ fields = [item for pair in record.items() for item in pair]+ outcome = get_redis_connection("default").eval(+ _QUEUE_SCRIPT,+ 4,+ record_key,+ _active_key(user_id),+ READY_KEY,+ _domain_pages_key(record_key),+ settings.SEED_CRAWL_MAX_QUEUED,+ queued_at,+ settings.SEED_CRAWL_RECORD_TTL_SECONDS,+ *fields,+ )+ return CrawlOutcome(outcome.decode())def get_active_seed_crawl_query(user_id: int) -> str | None:"""The query of this user's queued or running crawl, or None if they have none."""- query = get_redis_connection("default").get(_active_key(user_id))- return None if query is None else query.decode()+ redis = get_redis_connection("default")+ record_key = redis.get(_active_key(user_id))+ if record_key is None:+ return None+ query = redis.hget(record_key, "query")+ return query.decode()def run_seed_crawl_worker() -> None:- """Run queued seed crawls, one at a time, for ever."""+ """Run queued seed crawls' rounds, oldest due first, for ever.++ Stopping it at any point, however abruptly, loses at most the round it was on, which the+ next claim of that crawl runs again.+ """index_path = str(Path(settings.DATA_PATH) / settings.INDEX_NAME)while True:- run_next_seed_crawl(index_path)+ if not run_next_seed_crawl_round(index_path):+ time.sleep(POLL_SECONDS)+++# Claims the crawl due soonest, if one is due, by pushing its score out to when the lease+# ends, and returns its record's key and a fresh token. A crawl whose round has been claimed+# SEED_CRAWL_MAX_ATTEMPTS times without finishing is failed instead: the round may be what+# keeps killing its workers. The user's active crawl key is built here rather than passed+# in, as the crawl is only known once claimed.+_CLAIM_SCRIPT = """+local ready = KEYS[1]+local due = redis.call('ZRANGEBYSCORE', ready, '-inf', ARGV[1], 'LIMIT', 0, 1)+local record = due[1]+if not record then+ return false+end+local attempts = redis.call('HINCRBY', record, 'attempts', 1)+local token = redis.call('HINCRBY', record, 'token', 1)+if attempts > tonumber(ARGV[3]) then+ redis.call('HSET', record, 'status', 'failed', 'finished_at', ARGV[4])+ redis.call('HDEL', record, 'frontier')+ redis.call('ZREM', ready, record)+ redis.call('DEL', 'seed-crawl:' .. redis.call('HGET', record, 'user_id') .. ':active-crawl')+ return false+end+if redis.call('HGET', record, 'status') == 'queued' then+ redis.call('HSET', record, 'status', 'crawling', 'crawl_started_at', ARGV[4])+end+redis.call('ZADD', ready, tonumber(ARGV[1]) + tonumber(ARGV[2]), record)+return {record, token}+"""-def run_next_seed_crawl(index_path: str) -> bool:- """Run the next queued seed crawl, if one arrives within QUEUE_POLL_SECONDS."""+def run_next_seed_crawl_round(index_path: str) -> bool:+ """Run a round of the crawl due soonest, returning whether one was due."""redis = get_redis_connection("default")- popped = redis.brpop(QUEUE_KEY, timeout=QUEUE_POLL_SECONDS)- if popped is None:+ claimed = redis.eval(+ _CLAIM_SCRIPT,+ 1,+ READY_KEY,+ time.time(),+ settings.SEED_CRAWL_LEASE_SECONDS,+ settings.SEED_CRAWL_MAX_ATTEMPTS,+ _now(),+ )+ if claimed is None:return False+ record_key = claimed[0].decode()+ token = claimed[1]# A worker that waits for hours between crawls would otherwise reuse a connection# Postgres has long since closed.close_old_connections()- job = json.loads(popped[1])- # Jobs queued before seed domains listed their domains without Staan's result counts.- domains = job["domains"] if isinstance(job["domains"], dict) else dict.fromkeys(job["domains"], 1)- run_seed_crawl(job["user_id"], job["query"], job["seed_urls"], job["new_seed_urls"], domains, index_path)- return True---def run_seed_crawl(- user_id: int, query: str, seed_urls: list[str], new_seed_urls: list[str], domains: dict[str, int], index_path: str-) -> None:- redis = get_redis_connection("default")- record_key = _record_key(user_id, query)- redis.hset(record_key, mapping={"status": STATUS_CRAWLING, "crawl_started_at": _now()})- # A query's Staan results count towards its domains once per user, however often they- # crawl it, so nobody can raise a domain's score by repeating a query.- first_crawl_of_query = bool(redis.hsetnx(record_key, "staan_results_counted", 1))+ fields = {key.decode(): value.decode() for key, value in redis.hgetall(record_key).items()}+ user_id = int(fields["user_id"])try:- discovered = register_seed_domains(user_id, domains, count_staan_results=first_crawl_of_query)- redis.hset(record_key, "discovered", json.dumps(sorted(discovered)))- domain_pages = _crawl_and_record(- user_id, query, seed_urls, set(new_seed_urls), list(domains), index_path, redis- )- # Only a finished crawl is recorded: a failed one's counts would understate its domains.- record_seed_domain_crawls({domain: domain_pages[domain] for domain in domains})- status = STATUS_DONE+ _run_round(redis, record_key, token, fields, index_path)except Exception:- # Recorded rather than retried: a retry would start again from the seeds, fetching- # every site a second time. The worker carries on with the next crawl.+ # Failed rather than retried: whatever went wrong would most likely go wrong again.logger.exception("Seed crawl for user %d failed", user_id)- status = STATUS_FAILED+ _finish(redis, record_key, token, user_id, STATUS_FAILED)+ return True- # The user is freed only once the status is written, so a crawl they queue straight away- # cannot have its fresh record overwritten with this one's outcome.- pipeline = redis.pipeline()- pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})- pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)- pipeline.delete(_active_key(user_id))- pipeline.execute()+def _write_if_still_claimed(redis, record_key: str, token: int, write: Callable[[Pipeline], None]) -> bool:+ """Make write's changes in one transaction if the crawl is still this claim's, returning whether it was.-def _crawl_and_record(- user_id: int, query: str, seed_urls: list[str], new_seed_urls: set[str], domains: list[str], index_path: str, redis-) -> Counter:- """Crawl, index and record what is new, returning how many new pages each domain gave."""- record_key = _record_key(user_id, query)- pages_key = _pages_key(user_id, query)- counted_urls_key = _counted_urls_key(user_id, query)- domain_pages_key = _domain_pages_key(user_id, query)- ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS- domain_pages = Counter()-- rounds = crawl_within_domains(seed_urls, set(domains), redis)- # Each round is told which of its pages were new, so it can tell when a domain has run dry.- counted_urls = None- while True:+ The token changes whenever the crawl is claimed again or queued afresh, so a worker that+ has been too slow, or whose crawl has since been failed, writes nothing.+ """+ with redis.pipeline() as pipeline:try:- num_crawled, documents = rounds.send(counted_urls)- except StopIteration:- return domain_pages- indexed = index_new_documents(documents, index_path)- # Combined Search has since written Staan's snippets of the new seeds, so the index- # alone no longer says they were new. Only those this write stored count.- new_urls = indexed.new | (indexed.stored & new_seed_urls)- candidates = [document for document in documents if document.url in new_urls]-- # A page is counted once per query, however many crawls of it find it new.- pipeline = redis.pipeline()- for document in candidates:- pipeline.sadd(counted_urls_key, document.url)- first_counted = pipeline.execute()- new_documents = [document for document, added in zip(candidates, first_counted) if added]- logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))- round_domain_pages = Counter(bare_host(document.url) for document in new_documents)- domain_pages.update(round_domain_pages)-- pipeline = redis.pipeline()- pipeline.hincrby(record_key, "pages_crawled", num_crawled)- pipeline.hincrby(record_key, "rounds", 1)- if new_documents:- pages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]- pipeline.rpush(pages_key, *[json.dumps(page) for page in pages])- pipeline.expire(pages_key, ttl)- pipeline.expire(counted_urls_key, ttl)- for domain, count in round_domain_pages.items():- pipeline.hincrby(domain_pages_key, domain, count)- pipeline.expire(domain_pages_key, ttl)- pipeline.execute()-- if new_documents:- MwmblUser.objects.filter(id=user_id).update(- seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)- )- counted_urls = {document.url for document in new_documents}+ pipeline.watch(record_key)+ if int(pipeline.hget(record_key, "token")) != token:+ return False+ pipeline.multi()+ write(pipeline)+ pipeline.execute()+ except WatchError:+ return False+ return True-def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):- """Crawl breadth first from the seeds, staying within the domains given.+def _run_round(redis, record_key: str, token: int, fields: dict[str, str], index_path: str) -> None:+ user_id = int(fields["user_id"])+ domains = json.loads(fields["domains"])- Yields, for each round, how many URLs it fetched and the documents it got from them, and- is sent back the URLs of those documents that were new. A domain is dropped once- SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN fetches in a row from it have added nothing new, so- the crawl ends when there is nothing left for it to find rather than when it has fetched- every domain's budget.- """- frontiers: dict[str, deque[str]] = {}- seen_urls = set()- enqueued_per_domain = Counter()- stale_fetches_per_domain = Counter()- max_pages_per_domain = settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN-- def enqueue(url: str) -> None:- domain = bare_host(url)- if url in seen_urls or domain not in domains or enqueued_per_domain[domain] >= max_pages_per_domain:+ if "discovered" not in fields:+ # A query's Staan results count towards its domains once per user, however often+ # they crawl it, so nobody can raise a domain's score by repeating a query.+ first_crawl_of_query = bool(redis.hsetnx(record_key, "staan_results_counted", 1))+ discovered = register_seed_domains(user_id, domains, count_staan_results=first_crawl_of_query)+ discovered_json = json.dumps(sorted(discovered))+ if not _write_if_still_claimed(+ redis, record_key, token, lambda pipeline: pipeline.hset(record_key, "discovered", discovered_json)+ ):return- seen_urls.add(url)- enqueued_per_domain[domain] += 1- frontiers.setdefault(domain, deque()).append(url)-- for url in seed_urls:- enqueue(url)- def follow(link: str) -> None:- # Query strings are where faceted search, sorting and calendars multiply one page- # into thousands, which would spend a domain's budget on near-duplicates.- if urlsplit(link).query:- return- enqueue(link)-- # The per-domain cap already bounds the rounds; this bounds a round's slow fetches.- deadline = time.monotonic() + settings.SEED_CRAWL_MAX_SECONDS- last_round_started = None- while time.monotonic() < deadline:- batch = [frontier.popleft() for frontier in frontiers.values() if frontier]- if not batch:- return+ frontier = Frontier.from_json(fields["frontier"])+ crawl_started_at = datetime.fromisoformat(fields["crawl_started_at"])+ elapsed_seconds = (datetime.now(timezone.utc) - crawl_started_at).total_seconds()+ # The per-domain cap already bounds the rounds; this bounds their slow fetches.+ batch = frontier.next_batch() if elapsed_seconds < settings.SEED_CRAWL_MAX_SECONDS else []+ if not batch:+ _finish(redis, record_key, token, user_id, STATUS_DONE, domains)+ return++ if "last_round_started" in fields:+ # One crawl at a time, so this keeps any one site from being fetched from faster.+ next_round_due = float(fields["last_round_started"]) + settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS+ time.sleep(max(0.0, next_round_due - time.time()))+ round_started = time.time()+ documents = crawl_round(frontier, batch, redis)+ # Should the worker die between indexing the round and recording it, the round is run+ # again, and finds its pages already indexed: they are lost from the record, not counted twice.+ indexed = index_new_documents(documents, index_path)+ # Combined Search has since written Staan's snippets of the new seeds, so the index alone+ # no longer says they were new. Only those this write stored count.+ new_seed_urls = set(json.loads(fields["new_seed_urls"]))+ new_urls = indexed.new | (indexed.stored & new_seed_urls)+ candidates = [document for document in documents if document.url in new_urls]++ # A page is counted once per query, however many crawls of it find it new.+ counted_urls_key = _counted_urls_key(record_key)+ pipeline = redis.pipeline()+ for document in candidates:+ pipeline.sismember(counted_urls_key, document.url)+ already_counted = pipeline.execute()+ new_documents = [document for document, counted in zip(candidates, already_counted) if not counted]+ frontier.record_round(batch, {document.url for document in new_documents})+ round_domain_pages = Counter(bare_host(document.url) for document in new_documents)++ def write(pipeline: Pipeline) -> None:+ ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS+ pipeline.hset(+ record_key,+ mapping={"frontier": frontier.to_json(), "attempts": 0, "last_round_started": round_started},+ )+ pipeline.hincrby(record_key, "pages_crawled", len(batch))+ pipeline.hincrby(record_key, "rounds", 1)+ pipeline.expire(record_key, ttl)+ if new_documents:+ pages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]+ pipeline.sadd(counted_urls_key, *[document.url for document in new_documents])+ pipeline.expire(counted_urls_key, ttl)+ pipeline.rpush(_pages_key(record_key), *[json.dumps(page) for page in pages])+ pipeline.expire(_pages_key(record_key), ttl)+ for domain, count in round_domain_pages.items():+ pipeline.hincrby(_domain_pages_key(record_key), domain, count)+ pipeline.expire(_domain_pages_key(record_key), ttl)+ # Released, and still ahead of every crawl queued after it.+ pipeline.zadd(READY_KEY, {record_key: float(fields["queued_at"])})++ if not _write_if_still_claimed(redis, record_key, token, write):+ return+ logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))+ # After the round is recorded, so a round run twice is not counted twice.+ if new_documents:+ MwmblUser.objects.filter(id=user_id).update(+ seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)+ )- if last_round_started is not None:- elapsed = time.monotonic() - last_round_started- time.sleep(max(0.0, settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS - elapsed))- last_round_started = time.monotonic()- results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)+def _finish(+ redis, record_key: str, token: int, user_id: int, status: str, domains: dict[str, int] | None = None+) -> None:+ """End the crawl with this status, freeing the user for another, then score its domains if it is done."""- documents = []- for result in results:- content = result["content"]- # An error page is not content, and its links are no lead to any.- if content is None or result["status"] != HTTPStatus.OK:- continue- if content["title"]:- documents.append(- Document(- content["title"],- result["url"],- content["extract"],- last_crawled=result["timestamp"] // 1000,- )- )- for link in content["links"] + content["extra_links"]:- follow(link)- new_urls = yield len(batch), documents+ def write(pipeline: Pipeline) -> None:+ pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})+ pipeline.hdel(record_key, "frontier")+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)+ pipeline.zrem(READY_KEY, record_key)+ pipeline.delete(_active_key(user_id))- for url, result in zip(batch, results):- domain = bare_host(url)- if result["url"] in new_urls:- stale_fetches_per_domain[domain] = 0- continue- stale_fetches_per_domain[domain] += 1- if stale_fetches_per_domain[domain] >= settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN:- frontiers[domain].clear()+ if not _write_if_still_claimed(redis, record_key, token, write) or status != STATUS_DONE:+ return+ # Only a finished crawl is recorded: a failed one's counts would understate its domains.+ domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(record_key)).items()}+ record_seed_domain_crawls({domain: domain_pages.get(domain, 0) for domain in domains})def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:@@ -371,11 +518,16 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:thousands."""redis = get_redis_connection("default")- record = redis.hgetall(_record_key(user_id, query))+ record_key = _record_key(user_id, query)+ record = redis.hgetall(record_key)if not record:return Nonefields = {key.decode(): value.decode() for key, value in record.items()}- domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(user_id, query)).items()}+ # Every crawl queued or running is in the ready set. Those of crawls queued before they+ # were leased, whose workers died, are not, and are no longer being crawled.+ if fields["status"] in (STATUS_QUEUED, STATUS_CRAWLING) and redis.zscore(READY_KEY, record_key) is None:+ fields["status"] = STATUS_FAILED+ domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(record_key)).items()}# Records written before seed domains existed have no domains; they expire within a week.domains = list(json.loads(fields.get("domains", "{}")))# Absent until the worker has registered the crawl's domains, and with it whether each is@@ -389,7 +541,7 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:"started_at": fields["started_at"],"finished_at": fields.get("finished_at"),"pages_crawled": int(fields["pages_crawled"]),- "pages_indexed": redis.llen(_pages_key(user_id, query)),+ "pages_indexed": redis.llen(_pages_key(record_key)),"progress": _progress(fields),"domains": _domain_results(domains, discovered, domain_pages),}@@ -406,10 +558,8 @@ def _progress(fields: dict[str, str]) -> float:return 0.0if fields["status"] != STATUS_CRAWLING:return 1.0- # A record left crawling by a worker killed before progress was recorded has neither- # field; it expires within a week.- rounds_progress = int(fields.get("rounds", 0)) / settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN- crawl_started_at = fields.get("crawl_started_at", fields["started_at"])+ rounds_progress = int(fields["rounds"]) / settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN+ crawl_started_at = fields["crawl_started_at"]elapsed = datetime.now(timezone.utc) - datetime.fromisoformat(crawl_started_at)time_progress = elapsed.total_seconds() / settings.SEED_CRAWL_MAX_SECONDSreturn min(1.0, max(rounds_progress, time_progress))@@ -442,5 +592,5 @@ def get_seed_crawl(user_id: int, query: str) -> dict | None:if summary is None:return Noneredis = get_redis_connection("default")- pages = [json.loads(page) for page in redis.lrange(_pages_key(user_id, query), 0, -1)]+ pages = [json.loads(page) for page in redis.lrange(_pages_key(_record_key(user_id, query)), 0, -1)]return {**summary, "pages": pages}diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex ebf0faa..d53375f 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -315,18 +315,23 @@ STAAN_TIMEOUT_SECONDS = 5# Seed crawls: Combined Search with crawl=true crawls the Staan results the index lacked,# following links within Staan's domains - see mwmbl.indexer.seed_crawl.-SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 100+SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 20# A domain is dropped from a crawl after this many fetches in a row that added no new page.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN = 10# A backstop on a crawl's length; the per-domain cap ends almost all of them well before it.SEED_CRAWL_MAX_SECONDS = 15 * 60-# Crawls waiting for the worker, across all users, beyond which crawl=true queues nothing.+# Crawls queued or running, across all users, beyond which crawl=true queues nothing.SEED_CRAWL_MAX_QUEUED = 10SEED_CRAWL_THREADS = 8# The least time between two fetches from one domain.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 1.0# Queries are private, so a user's record of what their crawl added is not kept for long.SEED_CRAWL_RECORD_TTL_SECONDS = 7 * 24 * 60 * 60+# How long a worker has to finish a crawl's round before another may claim it. Far longer+# than a round takes - one fetch per domain, each with a three-second timeout, then indexing.+SEED_CRAWL_LEASE_SECONDS = 120+# Claims of one round, by workers that died before finishing it, after which the crawl fails.+SEED_CRAWL_MAX_ATTEMPTS = 3# A seed domain's new-page score averages only its most recent crawls, so a domain the# crawls have saturated drops down the list rather than living off its first crawl.SEED_DOMAIN_RECENT_CRAWLS = 5diff --git a/test/test_combined_search.py b/test/test_combined_search.pyindex de127f9..413c8b0 100644--- a/test/test_combined_search.py+++ b/test/test_combined_search.py@@ -530,14 +530,12 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(assert body["crawl_scheduled"] is Trueassert body["crawl_outcome"] == "scheduled"- job = json.loads(redis_cache.rpop(seed_crawl.QUEUE_KEY))- assert job == {- "user_id": user.id,- "query": "tokio",- "seed_urls": [STAAN_RESULT.url],- "new_seed_urls": [STAAN_RESULT.url],- "domains": {"tokio.rs": 1},- }+ record_key = seed_crawl._record_key(user.id, "tokio")+ assert redis_cache.zrange(seed_crawl.READY_KEY, 0, -1) == [record_key.encode()]+ record = {key.decode(): value.decode() for key, value in redis_cache.hgetall(record_key).items()}+ assert seed_crawl.Frontier.from_json(record["frontier"]).queues == {"tokio.rs": [STAAN_RESULT.url]}+ assert json.loads(record["new_seed_urls"]) == [STAAN_RESULT.url]+ assert json.loads(record["domains"]) == {"tokio.rs": 1}assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "queued"@@ -549,7 +547,7 @@ def test_no_crawl_without_the_flag(client, access_token, fresh_quota, redis_cachassert body["crawl_scheduled"] is Falseassert body["crawl_outcome"] is None- assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0@pytest.mark.django_db@@ -562,7 +560,7 @@ def test_no_crawl_when_the_index_already_has_every_staan_result(assert body["crawl_scheduled"] is Falseassert body["crawl_outcome"] == "already_indexed"- assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0@pytest.mark.django_dbdiff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.pyindex 253ddbd..9bae836 100644--- a/test/test_seed_crawl.py+++ b/test/test_seed_crawl.py@@ -65,11 +65,27 @@ def fake_web(monkeypatch, settings):return rounds-def worker(monkeypatch):- """The worker, minus its connection cleanup, which would close the test's transaction."""[email protected](autouse=True)+def no_connection_cleanup(monkeypatch):+ """The worker's connection cleanup would close the test's transaction."""monkeypatch.setattr(seed_crawl, "close_old_connections", lambda: None)- return seed_crawl.run_next_seed_crawl+++def _run_rounds(index_path):+ """Run the worker until no crawl is due, returning whether any was.++ The fake web has no delay between a domain's fetches, so a crawl is due again as soon as+ its round is done.+ """+ ran = False+ while seed_crawl.run_next_seed_crawl_round(index_path):+ ran = True+ return ran+++def worker():+ return _run_rounds@pytest.fixture@@ -86,17 +102,14 @@ def user(db):def _crawl(seed_urls, domains, is_new=lambda url: True):- """Run a crawl to its end, telling it that the pages is_new picks were new."""- rounds = seed_crawl.crawl_within_domains(seed_urls, set(domains), redis=None)+ """Run a crawl's rounds to the end, telling it that the pages is_new picks were new."""+ frontier = seed_crawl.Frontier.from_seeds(seed_urls, set(domains))crawled = []- new_urls = None- while True:- try:- num_crawled, documents = rounds.send(new_urls)- except StopIteration:- return crawled- crawled.append((num_crawled, documents))- new_urls = {document.url for document in documents if is_new(document.url)}+ while batch := frontier.next_batch():+ documents = seed_crawl.crawl_round(frontier, batch, redis=None)+ crawled.append((len(batch), documents))+ frontier.record_round(batch, {document.url for document in documents if is_new(document.url)})+ return crawled# ---------------------------------------------------------------------------@@ -187,7 +200,19 @@ def test_links_with_a_query_string_are_not_followed(fake_web):def _queued_jobs(redis_cache):- return [json.loads(job) for job in reversed(redis_cache.lrange(seed_crawl.QUEUE_KEY, 0, -1))]+ """The crawls queued or running, oldest first, with their seeds as yet unfetched."""+ jobs = []+ for record_key in redis_cache.zrange(seed_crawl.READY_KEY, 0, -1):+ record = {key.decode(): value.decode() for key, value in redis_cache.hgetall(record_key).items()}+ frontier = seed_crawl.Frontier.from_json(record["frontier"])+ jobs.append(+ {+ "seed_urls": [url for queue in frontier.queues.values() for url in queue],+ "new_seed_urls": json.loads(record["new_seed_urls"]),+ "domains": json.loads(record["domains"]),+ }+ )+ return jobsdef test_only_staan_results_missing_from_the_index_are_seeds(redis_cache, monkeypatch):@@ -273,10 +298,10 @@ def test_nothing_is_queued_once_the_queue_is_full(redis_cache, monkeypatch, settdef _run(user, index_path, seed_urls=(SEED,), new_seed_urls=()):- seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())- # Crawl only the seeds and domain given, whatever start_seed_crawl made of STAAN_RESULTS.- seed_crawl.get_redis_connection("default").delete(seed_crawl.QUEUE_KEY)- seed_crawl.run_seed_crawl(user.id, "tokio", list(seed_urls), list(new_seed_urls), {"tokio.rs": 1}, index_path)+ """Crawl from the seeds within tokio.rs to the end, as the user's crawl of "tokio"."""+ outcome = seed_crawl.queue_seed_crawl(user.id, "tokio", list(seed_urls), list(new_seed_urls), {"tokio.rs": 1})+ assert outcome == seed_crawl.CrawlOutcome.SCHEDULED+ _run_rounds(index_path)return seed_crawl.get_seed_crawl(user.id, "tokio")@@ -464,10 +489,10 @@ def test_a_finished_crawl_reports_its_domains_with_their_scores(fake_web, redis_"domain": "tokio.rs","newly_discovered": True,"pages_indexed": 3,- "new_page_score": 0.03,- "recent_new_page_score": 0.03,+ "new_page_score": 0.15,+ "recent_new_page_score": 0.15,"staan_results": 1,- "score": 0.03,+ "score": 0.15,}]@@ -489,11 +514,13 @@ def test_the_user_who_first_crawls_a_domain_is_its_discoverer(fake_web, redis_cadef test_a_domain_the_crawls_have_exhausted_loses_its_score(fake_web, redis_cache, user, index_path, settings):settings.SEED_DOMAIN_RECENT_CRAWLS = 2_run(user, index_path)- assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.03+ assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.15# The pages are no longer new, so these crawls add none.+ _mark_failed(user)_run(user, index_path)- assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.015+ assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.075+ _mark_failed(user)_run(user, index_path)seed_domain = SeedDomain.objects.get(domain="tokio.rs")assert seed_domain.new_page_score == 0.0@@ -529,7 +556,9 @@ def test_domains_are_not_reported_until_the_crawl_has_registered_them(fake_web,_run(user, index_path)# A crawl of a fresh query, picked up by the worker but not yet registered.seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set())- redis_cache.hset(seed_crawl._record_key(user.id, "rust"), "status", "crawling")+ redis_cache.hset(+ seed_crawl._record_key(user.id, "rust"), mapping={"status": "crawling", "crawl_started_at": seed_crawl._now()}+ )assert seed_crawl.get_seed_crawl_summary(user.id, "rust")["domains"] == []@@ -544,17 +573,16 @@ def test_crawling_a_query_again_reports_only_the_new_crawls_domains(fake_web, re[domain] = record["domains"]assert domain["newly_discovered"] is Falseassert domain["pages_indexed"] == 0- assert domain["recent_new_page_score"] == 0.015+ assert domain["recent_new_page_score"] == 0.075@pytest.mark.django_db-def test_a_querys_staan_results_count_once_per_user(fake_web, redis_cache, user, index_path):+def test_a_querys_staan_results_count_once_per_user(fake_web, redis_cache, user, index_path, worker):tokio_results = [*STAAN_RESULTS, Document("Tokio docs", "https://tokio.rs/docs", "", source=DocumentSource.STAAN)]for _ in range(2):seed_crawl.start_seed_crawl(user.id, "tokio", [], tokio_results, set())- job = json.loads(seed_crawl.get_redis_connection("default").rpop(seed_crawl.QUEUE_KEY))- seed_crawl.run_seed_crawl(user.id, "tokio", [SEED], [], job["domains"], index_path)+ worker(index_path)_mark_failed(user)assert SeedDomain.objects.get(domain="tokio.rs").staan_results == 2@@ -643,3 +671,121 @@ def test_index_new_documents_counts_nothing_the_blacklist_drops(index_path, monkindexed = index_new_documents([Document("Tokio", SEED, "Tokio.")], index_path)assert indexed == (set(), set())+++# ---------------------------------------------------------------------------+# Workers that die, stall or take turns+# ---------------------------------------------------------------------------+++def _claim(redis_cache):+ """Claim the crawl due soonest as a worker would, but run none of it."""+ return redis_cache.eval(+ seed_crawl._CLAIM_SCRIPT,+ 1,+ seed_crawl.READY_KEY,+ seed_crawl.time.time(),+ seed_crawl.settings.SEED_CRAWL_LEASE_SECONDS,+ seed_crawl.settings.SEED_CRAWL_MAX_ATTEMPTS,+ seed_crawl._now(),+ )+++def test_a_crawl_whose_worker_died_is_resumed_once_its_lease_runs_out(+ fake_web, redis_cache, user, index_path, worker, settings+):+ settings.SEED_CRAWL_LEASE_SECONDS = 0+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})+ # A worker claims the crawl and dies before writing anything.+ _claim(redis_cache)++ worker(index_path)++ record = seed_crawl.get_seed_crawl(user.id, "tokio")+ assert record["status"] == "done"+ assert record["pages_indexed"] == 3+++def test_a_crawl_is_not_claimed_again_while_its_lease_lasts(fake_web, redis_cache, user, index_path, worker):+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})+ _claim(redis_cache)++ assert worker(index_path) is False+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"+++def test_a_worker_that_lost_its_claim_writes_nothing(redis_cache, user, settings):+ settings.SEED_CRAWL_LEASE_SECONDS = 0+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})+ record_key, stale_token = _claim(redis_cache)+ # Too slow: another worker has claimed the crawl since.+ _claim(redis_cache)++ seed_crawl._finish(redis_cache, record_key.decode(), stale_token, user.id, "failed")++ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"+ assert seed_crawl.get_active_seed_crawl_query(user.id) == "tokio"+++def test_a_round_that_keeps_killing_its_workers_fails_the_crawl(redis_cache, user, settings, monkeypatch):+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())+ settings.SEED_CRAWL_LEASE_SECONDS = 0+ settings.SEED_CRAWL_MAX_ATTEMPTS = 2+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})++ assert _claim(redis_cache) is not None+ assert _claim(redis_cache) is not None+ assert _claim(redis_cache) is None++ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0+ assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED+++def test_queued_crawls_run_one_after_another_oldest_first(fake_web, redis_cache, user, index_path, worker):+ other = User.objects.create_user(username="other", email="[email protected]", password="x")+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})+ seed_crawl.queue_seed_crawl(other.id, "tokio", ["https://tokio.rs/tutorial/spawning"], [], {"tokio.rs": 1})++ worker(index_path)++ # The first crawl to the end, then the second.+ assert fake_web == [+ [SEED],+ ["https://tokio.rs/tutorial"],+ ["https://tokio.rs/tutorial/spawning"],+ ["https://tokio.rs/tutorial/spawning"],+ ]+ assert seed_crawl.get_seed_crawl(other.id, "tokio")["status"] == "done"+++def test_a_crawls_rounds_are_spaced_by_the_domain_delay(fake_web, redis_cache, user, index_path, settings, monkeypatch):+ settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 60+ sleeps = []+ monkeypatch.setattr(seed_crawl.time, "sleep", sleeps.append)+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})++ seed_crawl.run_next_seed_crawl_round(index_path)+ seed_crawl.run_next_seed_crawl_round(index_path)++ assert sleeps == [pytest.approx(60, abs=1)]+++def test_a_crawl_left_running_before_crawls_were_leased_is_reported_failed_and_can_be_redone(+ redis_cache, user, monkeypatch+):+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())+ redis_cache.hset(+ seed_crawl._record_key(user.id, "tokio"),+ mapping={"status": "crawling", "query": "tokio", "started_at": seed_crawl._now(), "pages_crawled": 4},+ )++ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"+ assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED