Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Run seed crawls as leased, resumable rounds

Daoud Clarkec61847feab81
diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.py
index f2073a8..f7aca8f 100644
--- a/mwmbl/indexer/seed_crawl.py
+++ b/mwmbl/indexer/seed_crawl.py
@@ -12,21 +12,34 @@ pages, which also bounds a crawl to that many rounds - a few minutes - whatever
of domains. Each round is indexed as it finishes, so what a crawl has added is visible while
it is still running.
+Everything about a crawl is in one Redis hash, its record, which is read by
+GET /api/v2/combined-search/new-pages (and /new-pages/count). That includes its frontier,
+so a crawl is a series of rounds that any worker can pick up, rather than minutes of state
+held in one process. A crawl queued or running is in READY_KEY, scored by when it was
+queued, so the oldest is crawled to the end before the next starts. A worker claims it by
+pushing its score out by SEED_CRAWL_LEASE_SECONDS, runs one round, and writes the round's
+outcome together with the new frontier and the crawl's own score back - or, at the end, the
+crawl's status - in one transaction. A worker
+that dies mid-round never writes, so once its lease runs out the crawl is claimed again
+from the round it was on; one that is merely slow is shut out by the claim's token, which
+the next claim changes. So there is nothing to clean up after a worker, however it stopped.
+
The crawls run in their own process rather than on the django-background-tasks queue: they
take minutes, and in series with the periodic tasks they would hold up the usage sync and
-the Polar report. That process runs them one at a time, each user has at most one queued or
-running, and at most SEED_CRAWL_MAX_QUEUED wait, so nobody can fill the queue.
+the Polar report. One crawl runs at a time, each user has at most one
+queued or running, and at most SEED_CRAWL_MAX_QUEUED are, so nobody can fill the queue.
-What it added is recorded in Redis against the user and the query, where
-GET /api/v2/combined-search/new-pages (and /new-pages/count) reads it, and counted on the user for their stats.
-That record also says how far along the crawl is, and what it found on each of Staan's
-domains, which are scored for crawlers as seed domains - see mwmbl.indexer.seed_domains.
+The new pages a crawl added are counted on the user for their stats, and what it found on
+each of Staan's domains is scored for crawlers as seed domains - see
+mwmbl.indexer.seed_domains.
"""
import hashlib
import json
import time
-from collections import Counter, deque
+from collections import Counter
+from collections.abc import Callable
+from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import StrEnum
from http import HTTPStatus
@@ -38,6 +51,8 @@ from django.conf import settings
from django.db import close_old_connections
from django.db.models import F
from django_redis import get_redis_connection
+from redis import WatchError
+from redis.client import Pipeline
from mwmbl.crawler.retrieve import crawl_batch
from mwmbl.indexer.index_batches import index_new_documents
@@ -66,9 +81,11 @@ class CrawlOutcome(StrEnum):
QUEUE_FULL = "queue_full"
-QUEUE_KEY = "seed-crawl:queue"
-# Under the Redis client's five-second socket timeout, which a longer blocking pop would trip.
-QUEUE_POLL_SECONDS = 1
+# The records of the crawls queued or running, scored by when each was queued, or while a
+# worker holds one, by when its lease ends.
+READY_KEY = "seed-crawl:ready"
+# How long the worker waits before looking again when no crawl is due.
+POLL_SECONDS = 1
def _record_key(user_id: int, query: str) -> str:
@@ -77,26 +94,130 @@ def _record_key(user_id: int, query: str) -> str:
return f"seed-crawl:{user_id}:{query_hash}"
-def _pages_key(user_id: int, query: str) -> str:
- return f"{_record_key(user_id, query)}:pages"
+def _pages_key(record_key: str) -> str:
+ return f"{record_key}:pages"
-def _counted_urls_key(user_id: int, query: str) -> str:
- return f"{_record_key(user_id, query)}:urls"
+def _counted_urls_key(record_key: str) -> str:
+ return f"{record_key}:urls"
-def _domain_pages_key(user_id: int, query: str) -> str:
- return f"{_record_key(user_id, query)}:domain-pages"
+def _domain_pages_key(record_key: str) -> str:
+ return f"{record_key}:domain-pages"
def _active_key(user_id: int) -> str:
- return f"seed-crawl:{user_id}:active"
+ # Named apart from the expiring lock that held the user's query before crawls were
+ # leased, so those left behind by the deploy cannot block anyone.
+ return f"seed-crawl:{user_id}:active-crawl"
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
+@dataclass
+class Frontier:
+ """What is left of a crawl: the URLs still to fetch from each domain, and its counts.
+
+ A domain is dropped once SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN fetches in a row from it
+ have added nothing new, so a crawl ends when there is nothing left for it to find rather
+ than when it has fetched every domain's budget.
+ """
+
+ domains: set[str]
+ queues: dict[str, list[str]] = field(default_factory=dict)
+ seen_urls: set[str] = field(default_factory=set)
+ enqueued_per_domain: Counter = field(default_factory=Counter)
+ stale_fetches_per_domain: Counter = field(default_factory=Counter)
+
+ @classmethod
+ def from_seeds(cls, seed_urls: list[str], domains: set[str]) -> "Frontier":
+ frontier = cls(domains)
+ for url in seed_urls:
+ frontier.enqueue(url)
+ return frontier
+
+ @classmethod
+ def from_json(cls, data: str) -> "Frontier":
+ fields = json.loads(data)
+ return cls(
+ set(fields["domains"]),
+ fields["queues"],
+ set(fields["seen_urls"]),
+ Counter(fields["enqueued_per_domain"]),
+ Counter(fields["stale_fetches_per_domain"]),
+ )
+
+ def to_json(self) -> str:
+ return json.dumps(
+ {
+ "domains": sorted(self.domains),
+ "queues": self.queues,
+ "seen_urls": sorted(self.seen_urls),
+ "enqueued_per_domain": self.enqueued_per_domain,
+ "stale_fetches_per_domain": self.stale_fetches_per_domain,
+ }
+ )
+
+ def enqueue(self, url: str) -> None:
+ domain = bare_host(url)
+ if (
+ url in self.seen_urls
+ or domain not in self.domains
+ or self.enqueued_per_domain[domain] >= settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN
+ ):
+ return
+ self.seen_urls.add(url)
+ self.enqueued_per_domain[domain] += 1
+ self.queues.setdefault(domain, []).append(url)
+
+ def follow(self, link: str) -> None:
+ # Query strings are where faceted search, sorting and calendars multiply one page
+ # into thousands, which would spend a domain's budget on near-duplicates.
+ if urlsplit(link).query:
+ return
+ self.enqueue(link)
+
+ def next_batch(self) -> list[str]:
+ """Take the next URL of each domain that has one."""
+ return [queue.pop(0) for queue in self.queues.values() if queue]
+
+ def record_round(self, batch: list[str], new_urls: set[str]) -> None:
+ """Note which of a round's URLs gave new pages, dropping the domains that have run dry."""
+ for url in batch:
+ domain = bare_host(url)
+ if url in new_urls:
+ self.stale_fetches_per_domain[domain] = 0
+ continue
+ self.stale_fetches_per_domain[domain] += 1
+ if self.stale_fetches_per_domain[domain] >= settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN:
+ self.queues[domain].clear()
+
+
+def crawl_round(frontier: Frontier, batch: list[str], redis) -> list[Document]:
+ """Fetch a round's URLs, adding their links to the frontier and returning their documents."""
+ results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)
+ documents = []
+ for result in results:
+ content = result["content"]
+ # An error page is not content, and its links are no lead to any.
+ if content is None or result["status"] != HTTPStatus.OK:
+ continue
+ if content["title"]:
+ documents.append(
+ Document(
+ content["title"],
+ result["url"],
+ content["extract"],
+ last_crawled=result["timestamp"] // 1000,
+ )
+ )
+ for link in content["links"] + content["extra_links"]:
+ frontier.follow(link)
+ return documents
+
+
def start_seed_crawl(
user_id: int, query: str, index_pages: list[Document], staan_results: list[Document], new_staan_urls: set[str]
) -> CrawlOutcome:
@@ -119,249 +240,275 @@ def start_seed_crawl(
if not seed_urls:
return CrawlOutcome.ALREADY_INDEXED
- redis = get_redis_connection("default")
- record_key = _record_key(user_id, query)
- # Pages a crawl adds need not rank for the query, so its seeds would mostly be the same
- # again: a query is crawled once while its record lasts, unless that crawl failed.
- status = redis.hget(record_key, "status")
- if status in (STATUS_QUEUED.encode(), STATUS_CRAWLING.encode()):
- return CrawlOutcome.ALREADY_RUNNING
- if status == STATUS_DONE.encode():
- return CrawlOutcome.ALREADY_CRAWLED
- # A slight overshoot from racing requests is harmless; the cap is there to stop one
- # user, or many, queueing hours of crawling.
- if redis.llen(QUEUE_KEY) >= settings.SEED_CRAWL_MAX_QUEUED:
- return CrawlOutcome.QUEUE_FULL
- # Set atomically, so a double click or a retried request cannot queue the crawl twice.
- # The expiry outlasts the longest wait plus the crawl, and frees the user should the
- # worker die mid-crawl.
- lock_seconds = settings.SEED_CRAWL_MAX_SECONDS * (settings.SEED_CRAWL_MAX_QUEUED + 1)
- if not redis.set(_active_key(user_id), query, nx=True, ex=lock_seconds):
- return CrawlOutcome.ALREADY_RUNNING
-
# Each domain with the number of Staan results it had, which the seed domains count.
staan_results_per_domain = Counter(bare_host(document.url) for document in allowed)
domains = dict(sorted(staan_results_per_domain.items()))
new_seed_urls = [url for url in seed_urls if url in new_staan_urls]
- job = {
- "user_id": user_id,
- "query": query,
- "seed_urls": seed_urls,
- "new_seed_urls": new_seed_urls,
- "domains": domains,
- }
- # A failed earlier crawl's pages are kept: the new one adds to them, and the user's total
- # already includes them. Its progress and domains are this crawl's alone.
+ return queue_seed_crawl(user_id, query, seed_urls, new_seed_urls, domains)
+
+
+# Queues a crawl, unless the user already has one queued or running, has crawled the query
+# already, or the queue is full. One script, so the user's active crawl, the ready set and
+# the record can never disagree. A failed earlier crawl's pages are kept: the new one adds
+# to them, and the user's total already includes them. Its progress and domains are this
+# crawl's alone. The token is bumped so that no worker still holding an earlier crawl of the
+# query can write to this one.
+_QUEUE_SCRIPT = """
+local record, active, ready, domain_pages = KEYS[1], KEYS[2], KEYS[3], KEYS[4]
+if redis.call('ZSCORE', ready, record) then
+ return 'already_running'
+end
+if redis.call('HGET', record, 'status') == 'done' then
+ return 'already_crawled'
+end
+if redis.call('EXISTS', active) == 1 then
+ return 'already_running'
+end
+if redis.call('ZCARD', ready) >= tonumber(ARGV[1]) then
+ return 'queue_full'
+end
+redis.call('HSET', record, unpack(ARGV, 4))
+redis.call('HSETNX', record, 'pages_crawled', 0)
+redis.call('HDEL', record, 'finished_at', 'crawl_started_at', 'discovered', 'attempts', 'last_round_started')
+redis.call('HINCRBY', record, 'token', 1)
+redis.call('EXPIRE', record, ARGV[3])
+redis.call('DEL', domain_pages)
+redis.call('SET', active, record)
+redis.call('ZADD', ready, ARGV[2], record)
+return 'scheduled'
+"""
+
+
+def queue_seed_crawl(
+ user_id: int, query: str, seed_urls: list[str], new_seed_urls: list[str], domains: dict[str, int]
+) -> CrawlOutcome:
+ """Queue a crawl from these seeds within these domains, unless the user may not have one."""
+ record_key = _record_key(user_id, query)
+ queued_at = time.time()
record = {
"status": STATUS_QUEUED,
+ "user_id": user_id,
"query": query,
"started_at": _now(),
+ "queued_at": queued_at,
"rounds": 0,
"domains": json.dumps(domains),
+ "new_seed_urls": json.dumps(new_seed_urls),
+ "frontier": Frontier.from_seeds(seed_urls, set(domains)).to_json(),
}
- pipeline = redis.pipeline()
- pipeline.hset(record_key, mapping=record)
- pipeline.hsetnx(record_key, "pages_crawled", 0)
- pipeline.hdel(record_key, "finished_at", "crawl_started_at", "discovered")
- pipeline.delete(_domain_pages_key(user_id, query))
- pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)
- pipeline.lpush(QUEUE_KEY, json.dumps(job))
- pipeline.execute()
- return CrawlOutcome.SCHEDULED
+ fields = [item for pair in record.items() for item in pair]
+ outcome = get_redis_connection("default").eval(
+ _QUEUE_SCRIPT,
+ 4,
+ record_key,
+ _active_key(user_id),
+ READY_KEY,
+ _domain_pages_key(record_key),
+ settings.SEED_CRAWL_MAX_QUEUED,
+ queued_at,
+ settings.SEED_CRAWL_RECORD_TTL_SECONDS,
+ *fields,
+ )
+ return CrawlOutcome(outcome.decode())
def get_active_seed_crawl_query(user_id: int) -> str | None:
"""The query of this user's queued or running crawl, or None if they have none."""
- query = get_redis_connection("default").get(_active_key(user_id))
- return None if query is None else query.decode()
+ redis = get_redis_connection("default")
+ record_key = redis.get(_active_key(user_id))
+ if record_key is None:
+ return None
+ query = redis.hget(record_key, "query")
+ return query.decode()
def run_seed_crawl_worker() -> None:
- """Run queued seed crawls, one at a time, for ever."""
+ """Run queued seed crawls' rounds, oldest due first, for ever.
+
+ Stopping it at any point, however abruptly, loses at most the round it was on, which the
+ next claim of that crawl runs again.
+ """
index_path = str(Path(settings.DATA_PATH) / settings.INDEX_NAME)
while True:
- run_next_seed_crawl(index_path)
+ if not run_next_seed_crawl_round(index_path):
+ time.sleep(POLL_SECONDS)
+
+
+# Claims the crawl due soonest, if one is due, by pushing its score out to when the lease
+# ends, and returns its record's key and a fresh token. A crawl whose round has been claimed
+# SEED_CRAWL_MAX_ATTEMPTS times without finishing is failed instead: the round may be what
+# keeps killing its workers. The user's active crawl key is built here rather than passed
+# in, as the crawl is only known once claimed.
+_CLAIM_SCRIPT = """
+local ready = KEYS[1]
+local due = redis.call('ZRANGEBYSCORE', ready, '-inf', ARGV[1], 'LIMIT', 0, 1)
+local record = due[1]
+if not record then
+ return false
+end
+local attempts = redis.call('HINCRBY', record, 'attempts', 1)
+local token = redis.call('HINCRBY', record, 'token', 1)
+if attempts > tonumber(ARGV[3]) then
+ redis.call('HSET', record, 'status', 'failed', 'finished_at', ARGV[4])
+ redis.call('HDEL', record, 'frontier')
+ redis.call('ZREM', ready, record)
+ redis.call('DEL', 'seed-crawl:' .. redis.call('HGET', record, 'user_id') .. ':active-crawl')
+ return false
+end
+if redis.call('HGET', record, 'status') == 'queued' then
+ redis.call('HSET', record, 'status', 'crawling', 'crawl_started_at', ARGV[4])
+end
+redis.call('ZADD', ready, tonumber(ARGV[1]) + tonumber(ARGV[2]), record)
+return {record, token}
+"""
-def run_next_seed_crawl(index_path: str) -> bool:
- """Run the next queued seed crawl, if one arrives within QUEUE_POLL_SECONDS."""
+def run_next_seed_crawl_round(index_path: str) -> bool:
+ """Run a round of the crawl due soonest, returning whether one was due."""
redis = get_redis_connection("default")
- popped = redis.brpop(QUEUE_KEY, timeout=QUEUE_POLL_SECONDS)
- if popped is None:
+ claimed = redis.eval(
+ _CLAIM_SCRIPT,
+ 1,
+ READY_KEY,
+ time.time(),
+ settings.SEED_CRAWL_LEASE_SECONDS,
+ settings.SEED_CRAWL_MAX_ATTEMPTS,
+ _now(),
+ )
+ if claimed is None:
return False
+ record_key = claimed[0].decode()
+ token = claimed[1]
# A worker that waits for hours between crawls would otherwise reuse a connection
# Postgres has long since closed.
close_old_connections()
- job = json.loads(popped[1])
- # Jobs queued before seed domains listed their domains without Staan's result counts.
- domains = job["domains"] if isinstance(job["domains"], dict) else dict.fromkeys(job["domains"], 1)
- run_seed_crawl(job["user_id"], job["query"], job["seed_urls"], job["new_seed_urls"], domains, index_path)
- return True
-
-
-def run_seed_crawl(
- user_id: int, query: str, seed_urls: list[str], new_seed_urls: list[str], domains: dict[str, int], index_path: str
-) -> None:
- redis = get_redis_connection("default")
- record_key = _record_key(user_id, query)
- redis.hset(record_key, mapping={"status": STATUS_CRAWLING, "crawl_started_at": _now()})
- # A query's Staan results count towards its domains once per user, however often they
- # crawl it, so nobody can raise a domain's score by repeating a query.
- first_crawl_of_query = bool(redis.hsetnx(record_key, "staan_results_counted", 1))
+ fields = {key.decode(): value.decode() for key, value in redis.hgetall(record_key).items()}
+ user_id = int(fields["user_id"])
try:
- discovered = register_seed_domains(user_id, domains, count_staan_results=first_crawl_of_query)
- redis.hset(record_key, "discovered", json.dumps(sorted(discovered)))
- domain_pages = _crawl_and_record(
- user_id, query, seed_urls, set(new_seed_urls), list(domains), index_path, redis
- )
- # Only a finished crawl is recorded: a failed one's counts would understate its domains.
- record_seed_domain_crawls({domain: domain_pages[domain] for domain in domains})
- status = STATUS_DONE
+ _run_round(redis, record_key, token, fields, index_path)
except Exception:
- # Recorded rather than retried: a retry would start again from the seeds, fetching
- # every site a second time. The worker carries on with the next crawl.
+ # Failed rather than retried: whatever went wrong would most likely go wrong again.
logger.exception("Seed crawl for user %d failed", user_id)
- status = STATUS_FAILED
+ _finish(redis, record_key, token, user_id, STATUS_FAILED)
+ return True
- # The user is freed only once the status is written, so a crawl they queue straight away
- # cannot have its fresh record overwritten with this one's outcome.
- pipeline = redis.pipeline()
- pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})
- pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)
- pipeline.delete(_active_key(user_id))
- pipeline.execute()
+def _write_if_still_claimed(redis, record_key: str, token: int, write: Callable[[Pipeline], None]) -> bool:
+ """Make write's changes in one transaction if the crawl is still this claim's, returning whether it was.
-def _crawl_and_record(
- user_id: int, query: str, seed_urls: list[str], new_seed_urls: set[str], domains: list[str], index_path: str, redis
-) -> Counter:
- """Crawl, index and record what is new, returning how many new pages each domain gave."""
- record_key = _record_key(user_id, query)
- pages_key = _pages_key(user_id, query)
- counted_urls_key = _counted_urls_key(user_id, query)
- domain_pages_key = _domain_pages_key(user_id, query)
- ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS
- domain_pages = Counter()
-
- rounds = crawl_within_domains(seed_urls, set(domains), redis)
- # Each round is told which of its pages were new, so it can tell when a domain has run dry.
- counted_urls = None
- while True:
+ The token changes whenever the crawl is claimed again or queued afresh, so a worker that
+ has been too slow, or whose crawl has since been failed, writes nothing.
+ """
+ with redis.pipeline() as pipeline:
try:
- num_crawled, documents = rounds.send(counted_urls)
- except StopIteration:
- return domain_pages
- indexed = index_new_documents(documents, index_path)
- # Combined Search has since written Staan's snippets of the new seeds, so the index
- # alone no longer says they were new. Only those this write stored count.
- new_urls = indexed.new | (indexed.stored & new_seed_urls)
- candidates = [document for document in documents if document.url in new_urls]
-
- # A page is counted once per query, however many crawls of it find it new.
- pipeline = redis.pipeline()
- for document in candidates:
- pipeline.sadd(counted_urls_key, document.url)
- first_counted = pipeline.execute()
- new_documents = [document for document, added in zip(candidates, first_counted) if added]
- logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))
- round_domain_pages = Counter(bare_host(document.url) for document in new_documents)
- domain_pages.update(round_domain_pages)
-
- pipeline = redis.pipeline()
- pipeline.hincrby(record_key, "pages_crawled", num_crawled)
- pipeline.hincrby(record_key, "rounds", 1)
- if new_documents:
- pages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]
- pipeline.rpush(pages_key, *[json.dumps(page) for page in pages])
- pipeline.expire(pages_key, ttl)
- pipeline.expire(counted_urls_key, ttl)
- for domain, count in round_domain_pages.items():
- pipeline.hincrby(domain_pages_key, domain, count)
- pipeline.expire(domain_pages_key, ttl)
- pipeline.execute()
-
- if new_documents:
- MwmblUser.objects.filter(id=user_id).update(
- seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)
- )
- counted_urls = {document.url for document in new_documents}
+ pipeline.watch(record_key)
+ if int(pipeline.hget(record_key, "token")) != token:
+ return False
+ pipeline.multi()
+ write(pipeline)
+ pipeline.execute()
+ except WatchError:
+ return False
+ return True
-def crawl_within_domains(seed_urls: list[str], domains: set[str], redis):
- """Crawl breadth first from the seeds, staying within the domains given.
+def _run_round(redis, record_key: str, token: int, fields: dict[str, str], index_path: str) -> None:
+ user_id = int(fields["user_id"])
+ domains = json.loads(fields["domains"])
- Yields, for each round, how many URLs it fetched and the documents it got from them, and
- is sent back the URLs of those documents that were new. A domain is dropped once
- SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN fetches in a row from it have added nothing new, so
- the crawl ends when there is nothing left for it to find rather than when it has fetched
- every domain's budget.
- """
- frontiers: dict[str, deque[str]] = {}
- seen_urls = set()
- enqueued_per_domain = Counter()
- stale_fetches_per_domain = Counter()
- max_pages_per_domain = settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN
-
- def enqueue(url: str) -> None:
- domain = bare_host(url)
- if url in seen_urls or domain not in domains or enqueued_per_domain[domain] >= max_pages_per_domain:
+ if "discovered" not in fields:
+ # A query's Staan results count towards its domains once per user, however often
+ # they crawl it, so nobody can raise a domain's score by repeating a query.
+ first_crawl_of_query = bool(redis.hsetnx(record_key, "staan_results_counted", 1))
+ discovered = register_seed_domains(user_id, domains, count_staan_results=first_crawl_of_query)
+ discovered_json = json.dumps(sorted(discovered))
+ if not _write_if_still_claimed(
+ redis, record_key, token, lambda pipeline: pipeline.hset(record_key, "discovered", discovered_json)
+ ):
return
- seen_urls.add(url)
- enqueued_per_domain[domain] += 1
- frontiers.setdefault(domain, deque()).append(url)
-
- for url in seed_urls:
- enqueue(url)
- def follow(link: str) -> None:
- # Query strings are where faceted search, sorting and calendars multiply one page
- # into thousands, which would spend a domain's budget on near-duplicates.
- if urlsplit(link).query:
- return
- enqueue(link)
-
- # The per-domain cap already bounds the rounds; this bounds a round's slow fetches.
- deadline = time.monotonic() + settings.SEED_CRAWL_MAX_SECONDS
- last_round_started = None
- while time.monotonic() < deadline:
- batch = [frontier.popleft() for frontier in frontiers.values() if frontier]
- if not batch:
- return
+ frontier = Frontier.from_json(fields["frontier"])
+ crawl_started_at = datetime.fromisoformat(fields["crawl_started_at"])
+ elapsed_seconds = (datetime.now(timezone.utc) - crawl_started_at).total_seconds()
+ # The per-domain cap already bounds the rounds; this bounds their slow fetches.
+ batch = frontier.next_batch() if elapsed_seconds < settings.SEED_CRAWL_MAX_SECONDS else []
+ if not batch:
+ _finish(redis, record_key, token, user_id, STATUS_DONE, domains)
+ return
+
+ if "last_round_started" in fields:
+ # One crawl at a time, so this keeps any one site from being fetched from faster.
+ next_round_due = float(fields["last_round_started"]) + settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS
+ time.sleep(max(0.0, next_round_due - time.time()))
+ round_started = time.time()
+ documents = crawl_round(frontier, batch, redis)
+ # Should the worker die between indexing the round and recording it, the round is run
+ # again, and finds its pages already indexed: they are lost from the record, not counted twice.
+ indexed = index_new_documents(documents, index_path)
+ # Combined Search has since written Staan's snippets of the new seeds, so the index alone
+ # no longer says they were new. Only those this write stored count.
+ new_seed_urls = set(json.loads(fields["new_seed_urls"]))
+ new_urls = indexed.new | (indexed.stored & new_seed_urls)
+ candidates = [document for document in documents if document.url in new_urls]
+
+ # A page is counted once per query, however many crawls of it find it new.
+ counted_urls_key = _counted_urls_key(record_key)
+ pipeline = redis.pipeline()
+ for document in candidates:
+ pipeline.sismember(counted_urls_key, document.url)
+ already_counted = pipeline.execute()
+ new_documents = [document for document, counted in zip(candidates, already_counted) if not counted]
+ frontier.record_round(batch, {document.url for document in new_documents})
+ round_domain_pages = Counter(bare_host(document.url) for document in new_documents)
+
+ def write(pipeline: Pipeline) -> None:
+ ttl = settings.SEED_CRAWL_RECORD_TTL_SECONDS
+ pipeline.hset(
+ record_key,
+ mapping={"frontier": frontier.to_json(), "attempts": 0, "last_round_started": round_started},
+ )
+ pipeline.hincrby(record_key, "pages_crawled", len(batch))
+ pipeline.hincrby(record_key, "rounds", 1)
+ pipeline.expire(record_key, ttl)
+ if new_documents:
+ pages = [{"url": doc.url, "title": doc.title, "extract": doc.extract} for doc in new_documents]
+ pipeline.sadd(counted_urls_key, *[document.url for document in new_documents])
+ pipeline.expire(counted_urls_key, ttl)
+ pipeline.rpush(_pages_key(record_key), *[json.dumps(page) for page in pages])
+ pipeline.expire(_pages_key(record_key), ttl)
+ for domain, count in round_domain_pages.items():
+ pipeline.hincrby(_domain_pages_key(record_key), domain, count)
+ pipeline.expire(_domain_pages_key(record_key), ttl)
+ # Released, and still ahead of every crawl queued after it.
+ pipeline.zadd(READY_KEY, {record_key: float(fields["queued_at"])})
+
+ if not _write_if_still_claimed(redis, record_key, token, write):
+ return
+ logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))
+ # After the round is recorded, so a round run twice is not counted twice.
+ if new_documents:
+ MwmblUser.objects.filter(id=user_id).update(
+ seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)
+ )
- if last_round_started is not None:
- elapsed = time.monotonic() - last_round_started
- time.sleep(max(0.0, settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS - elapsed))
- last_round_started = time.monotonic()
- results = crawl_batch(batch, settings.SEED_CRAWL_THREADS, 0, redis)
+def _finish(
+ redis, record_key: str, token: int, user_id: int, status: str, domains: dict[str, int] | None = None
+) -> None:
+ """End the crawl with this status, freeing the user for another, then score its domains if it is done."""
- documents = []
- for result in results:
- content = result["content"]
- # An error page is not content, and its links are no lead to any.
- if content is None or result["status"] != HTTPStatus.OK:
- continue
- if content["title"]:
- documents.append(
- Document(
- content["title"],
- result["url"],
- content["extract"],
- last_crawled=result["timestamp"] // 1000,
- )
- )
- for link in content["links"] + content["extra_links"]:
- follow(link)
- new_urls = yield len(batch), documents
+ def write(pipeline: Pipeline) -> None:
+ pipeline.hset(record_key, mapping={"status": status, "finished_at": _now()})
+ pipeline.hdel(record_key, "frontier")
+ pipeline.expire(record_key, settings.SEED_CRAWL_RECORD_TTL_SECONDS)
+ pipeline.zrem(READY_KEY, record_key)
+ pipeline.delete(_active_key(user_id))
- for url, result in zip(batch, results):
- domain = bare_host(url)
- if result["url"] in new_urls:
- stale_fetches_per_domain[domain] = 0
- continue
- stale_fetches_per_domain[domain] += 1
- if stale_fetches_per_domain[domain] >= settings.SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN:
- frontiers[domain].clear()
+ if not _write_if_still_claimed(redis, record_key, token, write) or status != STATUS_DONE:
+ return
+ # Only a finished crawl is recorded: a failed one's counts would understate its domains.
+ domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(record_key)).items()}
+ record_seed_domain_crawls({domain: domain_pages.get(domain, 0) for domain in domains})
def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:
@@ -371,11 +518,16 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:
thousands.
"""
redis = get_redis_connection("default")
- record = redis.hgetall(_record_key(user_id, query))
+ record_key = _record_key(user_id, query)
+ record = redis.hgetall(record_key)
if not record:
return None
fields = {key.decode(): value.decode() for key, value in record.items()}
- domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(user_id, query)).items()}
+ # Every crawl queued or running is in the ready set. Those of crawls queued before they
+ # were leased, whose workers died, are not, and are no longer being crawled.
+ if fields["status"] in (STATUS_QUEUED, STATUS_CRAWLING) and redis.zscore(READY_KEY, record_key) is None:
+ fields["status"] = STATUS_FAILED
+ domain_pages = {key.decode(): int(value) for key, value in redis.hgetall(_domain_pages_key(record_key)).items()}
# Records written before seed domains existed have no domains; they expire within a week.
domains = list(json.loads(fields.get("domains", "{}")))
# Absent until the worker has registered the crawl's domains, and with it whether each is
@@ -389,7 +541,7 @@ def get_seed_crawl_summary(user_id: int, query: str) -> dict | None:
"started_at": fields["started_at"],
"finished_at": fields.get("finished_at"),
"pages_crawled": int(fields["pages_crawled"]),
- "pages_indexed": redis.llen(_pages_key(user_id, query)),
+ "pages_indexed": redis.llen(_pages_key(record_key)),
"progress": _progress(fields),
"domains": _domain_results(domains, discovered, domain_pages),
}
@@ -406,10 +558,8 @@ def _progress(fields: dict[str, str]) -> float:
return 0.0
if fields["status"] != STATUS_CRAWLING:
return 1.0
- # A record left crawling by a worker killed before progress was recorded has neither
- # field; it expires within a week.
- rounds_progress = int(fields.get("rounds", 0)) / settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN
- crawl_started_at = fields.get("crawl_started_at", fields["started_at"])
+ rounds_progress = int(fields["rounds"]) / settings.SEED_CRAWL_MAX_PAGES_PER_DOMAIN
+ crawl_started_at = fields["crawl_started_at"]
elapsed = datetime.now(timezone.utc) - datetime.fromisoformat(crawl_started_at)
time_progress = elapsed.total_seconds() / settings.SEED_CRAWL_MAX_SECONDS
return min(1.0, max(rounds_progress, time_progress))
@@ -442,5 +592,5 @@ def get_seed_crawl(user_id: int, query: str) -> dict | None:
if summary is None:
return None
redis = get_redis_connection("default")
- pages = [json.loads(page) for page in redis.lrange(_pages_key(user_id, query), 0, -1)]
+ pages = [json.loads(page) for page in redis.lrange(_pages_key(_record_key(user_id, query)), 0, -1)]
return {**summary, "pages": pages}
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index ebf0faa..d53375f 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -315,18 +315,23 @@ STAAN_TIMEOUT_SECONDS = 5
# Seed crawls: Combined Search with crawl=true crawls the Staan results the index lacked,
# following links within Staan's domains - see mwmbl.indexer.seed_crawl.
-SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 100
+SEED_CRAWL_MAX_PAGES_PER_DOMAIN = 20
# A domain is dropped from a crawl after this many fetches in a row that added no new page.
SEED_CRAWL_MAX_STALE_PAGES_PER_DOMAIN = 10
# A backstop on a crawl's length; the per-domain cap ends almost all of them well before it.
SEED_CRAWL_MAX_SECONDS = 15 * 60
-# Crawls waiting for the worker, across all users, beyond which crawl=true queues nothing.
+# Crawls queued or running, across all users, beyond which crawl=true queues nothing.
SEED_CRAWL_MAX_QUEUED = 10
SEED_CRAWL_THREADS = 8
# The least time between two fetches from one domain.
SEED_CRAWL_DOMAIN_DELAY_SECONDS = 1.0
# Queries are private, so a user's record of what their crawl added is not kept for long.
SEED_CRAWL_RECORD_TTL_SECONDS = 7 * 24 * 60 * 60
+# How long a worker has to finish a crawl's round before another may claim it. Far longer
+# than a round takes - one fetch per domain, each with a three-second timeout, then indexing.
+SEED_CRAWL_LEASE_SECONDS = 120
+# Claims of one round, by workers that died before finishing it, after which the crawl fails.
+SEED_CRAWL_MAX_ATTEMPTS = 3
# A seed domain's new-page score averages only its most recent crawls, so a domain the
# crawls have saturated drops down the list rather than living off its first crawl.
SEED_DOMAIN_RECENT_CRAWLS = 5
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
index de127f9..413c8b0 100644
--- a/test/test_combined_search.py
+++ b/test/test_combined_search.py
@@ -530,14 +530,12 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(
assert body["crawl_scheduled"] is True
assert body["crawl_outcome"] == "scheduled"
- job = json.loads(redis_cache.rpop(seed_crawl.QUEUE_KEY))
- assert job == {
- "user_id": user.id,
- "query": "tokio",
- "seed_urls": [STAAN_RESULT.url],
- "new_seed_urls": [STAAN_RESULT.url],
- "domains": {"tokio.rs": 1},
- }
+ record_key = seed_crawl._record_key(user.id, "tokio")
+ assert redis_cache.zrange(seed_crawl.READY_KEY, 0, -1) == [record_key.encode()]
+ record = {key.decode(): value.decode() for key, value in redis_cache.hgetall(record_key).items()}
+ assert seed_crawl.Frontier.from_json(record["frontier"]).queues == {"tokio.rs": [STAAN_RESULT.url]}
+ assert json.loads(record["new_seed_urls"]) == [STAAN_RESULT.url]
+ assert json.loads(record["domains"]) == {"tokio.rs": 1}
assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "queued"
@@ -549,7 +547,7 @@ def test_no_crawl_without_the_flag(client, access_token, fresh_quota, redis_cach
assert body["crawl_scheduled"] is False
assert body["crawl_outcome"] is None
- assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0
+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0
@pytest.mark.django_db
@@ -562,7 +560,7 @@ def test_no_crawl_when_the_index_already_has_every_staan_result(
assert body["crawl_scheduled"] is False
assert body["crawl_outcome"] == "already_indexed"
- assert redis_cache.llen(seed_crawl.QUEUE_KEY) == 0
+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0
@pytest.mark.django_db
diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.py
index 253ddbd..9bae836 100644
--- a/test/test_seed_crawl.py
+++ b/test/test_seed_crawl.py
@@ -65,11 +65,27 @@ def fake_web(monkeypatch, settings):
return rounds
-def worker(monkeypatch):
- """The worker, minus its connection cleanup, which would close the test's transaction."""
[email protected](autouse=True)
+def no_connection_cleanup(monkeypatch):
+ """The worker's connection cleanup would close the test's transaction."""
monkeypatch.setattr(seed_crawl, "close_old_connections", lambda: None)
- return seed_crawl.run_next_seed_crawl
+
+
+def _run_rounds(index_path):
+ """Run the worker until no crawl is due, returning whether any was.
+
+ The fake web has no delay between a domain's fetches, so a crawl is due again as soon as
+ its round is done.
+ """
+ ran = False
+ while seed_crawl.run_next_seed_crawl_round(index_path):
+ ran = True
+ return ran
+
+
+def worker():
+ return _run_rounds
@pytest.fixture
@@ -86,17 +102,14 @@ def user(db):
def _crawl(seed_urls, domains, is_new=lambda url: True):
- """Run a crawl to its end, telling it that the pages is_new picks were new."""
- rounds = seed_crawl.crawl_within_domains(seed_urls, set(domains), redis=None)
+ """Run a crawl's rounds to the end, telling it that the pages is_new picks were new."""
+ frontier = seed_crawl.Frontier.from_seeds(seed_urls, set(domains))
crawled = []
- new_urls = None
- while True:
- try:
- num_crawled, documents = rounds.send(new_urls)
- except StopIteration:
- return crawled
- crawled.append((num_crawled, documents))
- new_urls = {document.url for document in documents if is_new(document.url)}
+ while batch := frontier.next_batch():
+ documents = seed_crawl.crawl_round(frontier, batch, redis=None)
+ crawled.append((len(batch), documents))
+ frontier.record_round(batch, {document.url for document in documents if is_new(document.url)})
+ return crawled
# ---------------------------------------------------------------------------
@@ -187,7 +200,19 @@ def test_links_with_a_query_string_are_not_followed(fake_web):
def _queued_jobs(redis_cache):
- return [json.loads(job) for job in reversed(redis_cache.lrange(seed_crawl.QUEUE_KEY, 0, -1))]
+ """The crawls queued or running, oldest first, with their seeds as yet unfetched."""
+ jobs = []
+ for record_key in redis_cache.zrange(seed_crawl.READY_KEY, 0, -1):
+ record = {key.decode(): value.decode() for key, value in redis_cache.hgetall(record_key).items()}
+ frontier = seed_crawl.Frontier.from_json(record["frontier"])
+ jobs.append(
+ {
+ "seed_urls": [url for queue in frontier.queues.values() for url in queue],
+ "new_seed_urls": json.loads(record["new_seed_urls"]),
+ "domains": json.loads(record["domains"]),
+ }
+ )
+ return jobs
def test_only_staan_results_missing_from_the_index_are_seeds(redis_cache, monkeypatch):
@@ -273,10 +298,10 @@ def test_nothing_is_queued_once_the_queue_is_full(redis_cache, monkeypatch, sett
def _run(user, index_path, seed_urls=(SEED,), new_seed_urls=()):
- seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set())
- # Crawl only the seeds and domain given, whatever start_seed_crawl made of STAAN_RESULTS.
- seed_crawl.get_redis_connection("default").delete(seed_crawl.QUEUE_KEY)
- seed_crawl.run_seed_crawl(user.id, "tokio", list(seed_urls), list(new_seed_urls), {"tokio.rs": 1}, index_path)
+ """Crawl from the seeds within tokio.rs to the end, as the user's crawl of "tokio"."""
+ outcome = seed_crawl.queue_seed_crawl(user.id, "tokio", list(seed_urls), list(new_seed_urls), {"tokio.rs": 1})
+ assert outcome == seed_crawl.CrawlOutcome.SCHEDULED
+ _run_rounds(index_path)
return seed_crawl.get_seed_crawl(user.id, "tokio")
@@ -464,10 +489,10 @@ def test_a_finished_crawl_reports_its_domains_with_their_scores(fake_web, redis_
"domain": "tokio.rs",
"newly_discovered": True,
"pages_indexed": 3,
- "new_page_score": 0.03,
- "recent_new_page_score": 0.03,
+ "new_page_score": 0.15,
+ "recent_new_page_score": 0.15,
"staan_results": 1,
- "score": 0.03,
+ "score": 0.15,
}
]
@@ -489,11 +514,13 @@ def test_the_user_who_first_crawls_a_domain_is_its_discoverer(fake_web, redis_ca
def test_a_domain_the_crawls_have_exhausted_loses_its_score(fake_web, redis_cache, user, index_path, settings):
settings.SEED_DOMAIN_RECENT_CRAWLS = 2
_run(user, index_path)
- assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.03
+ assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.15
# The pages are no longer new, so these crawls add none.
+ _mark_failed(user)
_run(user, index_path)
- assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.015
+ assert SeedDomain.objects.get(domain="tokio.rs").new_page_score == 0.075
+ _mark_failed(user)
_run(user, index_path)
seed_domain = SeedDomain.objects.get(domain="tokio.rs")
assert seed_domain.new_page_score == 0.0
@@ -529,7 +556,9 @@ def test_domains_are_not_reported_until_the_crawl_has_registered_them(fake_web,
_run(user, index_path)
# A crawl of a fresh query, picked up by the worker but not yet registered.
seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set())
- redis_cache.hset(seed_crawl._record_key(user.id, "rust"), "status", "crawling")
+ redis_cache.hset(
+ seed_crawl._record_key(user.id, "rust"), mapping={"status": "crawling", "crawl_started_at": seed_crawl._now()}
+ )
assert seed_crawl.get_seed_crawl_summary(user.id, "rust")["domains"] == []
@@ -544,17 +573,16 @@ def test_crawling_a_query_again_reports_only_the_new_crawls_domains(fake_web, re
[domain] = record["domains"]
assert domain["newly_discovered"] is False
assert domain["pages_indexed"] == 0
- assert domain["recent_new_page_score"] == 0.015
+ assert domain["recent_new_page_score"] == 0.075
@pytest.mark.django_db
-def test_a_querys_staan_results_count_once_per_user(fake_web, redis_cache, user, index_path):
+def test_a_querys_staan_results_count_once_per_user(fake_web, redis_cache, user, index_path, worker):
tokio_results = [*STAAN_RESULTS, Document("Tokio docs", "https://tokio.rs/docs", "", source=DocumentSource.STAAN)]
for _ in range(2):
seed_crawl.start_seed_crawl(user.id, "tokio", [], tokio_results, set())
- job = json.loads(seed_crawl.get_redis_connection("default").rpop(seed_crawl.QUEUE_KEY))
- seed_crawl.run_seed_crawl(user.id, "tokio", [SEED], [], job["domains"], index_path)
+ worker(index_path)
_mark_failed(user)
assert SeedDomain.objects.get(domain="tokio.rs").staan_results == 2
@@ -643,3 +671,121 @@ def test_index_new_documents_counts_nothing_the_blacklist_drops(index_path, monk
indexed = index_new_documents([Document("Tokio", SEED, "Tokio.")], index_path)
assert indexed == (set(), set())
+
+
+# ---------------------------------------------------------------------------
+# Workers that die, stall or take turns
+# ---------------------------------------------------------------------------
+
+
+def _claim(redis_cache):
+ """Claim the crawl due soonest as a worker would, but run none of it."""
+ return redis_cache.eval(
+ seed_crawl._CLAIM_SCRIPT,
+ 1,
+ seed_crawl.READY_KEY,
+ seed_crawl.time.time(),
+ seed_crawl.settings.SEED_CRAWL_LEASE_SECONDS,
+ seed_crawl.settings.SEED_CRAWL_MAX_ATTEMPTS,
+ seed_crawl._now(),
+ )
+
+
+def test_a_crawl_whose_worker_died_is_resumed_once_its_lease_runs_out(
+ fake_web, redis_cache, user, index_path, worker, settings
+):
+ settings.SEED_CRAWL_LEASE_SECONDS = 0
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+ # A worker claims the crawl and dies before writing anything.
+ _claim(redis_cache)
+
+ worker(index_path)
+
+ record = seed_crawl.get_seed_crawl(user.id, "tokio")
+ assert record["status"] == "done"
+ assert record["pages_indexed"] == 3
+
+
+def test_a_crawl_is_not_claimed_again_while_its_lease_lasts(fake_web, redis_cache, user, index_path, worker):
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+ _claim(redis_cache)
+
+ assert worker(index_path) is False
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"
+
+
+def test_a_worker_that_lost_its_claim_writes_nothing(redis_cache, user, settings):
+ settings.SEED_CRAWL_LEASE_SECONDS = 0
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+ record_key, stale_token = _claim(redis_cache)
+ # Too slow: another worker has claimed the crawl since.
+ _claim(redis_cache)
+
+ seed_crawl._finish(redis_cache, record_key.decode(), stale_token, user.id, "failed")
+
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "crawling"
+ assert seed_crawl.get_active_seed_crawl_query(user.id) == "tokio"
+
+
+def test_a_round_that_keeps_killing_its_workers_fails_the_crawl(redis_cache, user, settings, monkeypatch):
+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
+ settings.SEED_CRAWL_LEASE_SECONDS = 0
+ settings.SEED_CRAWL_MAX_ATTEMPTS = 2
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+
+ assert _claim(redis_cache) is not None
+ assert _claim(redis_cache) is not None
+ assert _claim(redis_cache) is None
+
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"
+ assert redis_cache.zcard(seed_crawl.READY_KEY) == 0
+ assert seed_crawl.start_seed_crawl(user.id, "rust", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED
+
+
+def test_queued_crawls_run_one_after_another_oldest_first(fake_web, redis_cache, user, index_path, worker):
+ other = User.objects.create_user(username="other", email="[email protected]", password="x")
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+ seed_crawl.queue_seed_crawl(other.id, "tokio", ["https://tokio.rs/tutorial/spawning"], [], {"tokio.rs": 1})
+
+ worker(index_path)
+
+ # The first crawl to the end, then the second.
+ assert fake_web == [
+ [SEED],
+ ["https://tokio.rs/tutorial"],
+ ["https://tokio.rs/tutorial/spawning"],
+ ["https://tokio.rs/tutorial/spawning"],
+ ]
+ assert seed_crawl.get_seed_crawl(other.id, "tokio")["status"] == "done"
+
+
+def test_a_crawls_rounds_are_spaced_by_the_domain_delay(fake_web, redis_cache, user, index_path, settings, monkeypatch):
+ settings.SEED_CRAWL_DOMAIN_DELAY_SECONDS = 60
+ sleeps = []
+ monkeypatch.setattr(seed_crawl.time, "sleep", sleeps.append)
+ seed_crawl.queue_seed_crawl(user.id, "tokio", [SEED], [], {"tokio.rs": 1})
+
+ seed_crawl.run_next_seed_crawl_round(index_path)
+ seed_crawl.run_next_seed_crawl_round(index_path)
+
+ assert sleeps == [pytest.approx(60, abs=1)]
+
+
+def test_a_crawl_left_running_before_crawls_were_leased_is_reported_failed_and_can_be_redone(
+ redis_cache, user, monkeypatch
+):
+ monkeypatch.setattr(seed_crawl, "find_blacklisted_urls", lambda documents: set())
+ redis_cache.hset(
+ seed_crawl._record_key(user.id, "tokio"),
+ mapping={"status": "crawling", "query": "tokio", "started_at": seed_crawl._now(), "pages_crawled": 4},
+ )
+
+ assert seed_crawl.get_seed_crawl(user.id, "tokio")["status"] == "failed"
+ assert seed_crawl.start_seed_crawl(user.id, "tokio", [], STAAN_RESULTS, set()) == seed_crawl.CrawlOutcome.SCHEDULED