An open source, non-profit web search engine
fix: botched merge forgot some lines
thiswillbeyourgithubea1c6f8fecff
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex cd5f4c3..5f316b8 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -11,9 +11,7 @@ import requestsfrom django.conf import settingsfrom redis import Redis-CRAWLER_VERSION: str = "0.2.0"--from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS, CRAWL_DELAY_SECONDS+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_DELAY_SECONDS, MWMBL_API_KEY, REDIS_URLfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -30,14 +28,17 @@ print("Mwmbl crawling statistics: https://mwmbl.org/stats")django.setup()-from mwmbl.indexer.update_urls import record_urls_in_databasefrom mwmbl.crawler.retrieve import crawl_batch, crawl_urlfrom mwmbl.crawler.batch import HashedBatch, Result, Resultsfrom mwmbl.indexer.index_batches import index_batches, index_pages-API_KEY = MWMBL_API_KEY+logger = logging.getLogger(__name__)+FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"+logging.basicConfig(level=logging.INFO, format=FORMAT)+BATCH_QUEUE_KEY = "batch-queue"+CRAWLER_VERSION: str = "0.2.0"redis = Redis.from_url(@@ -71,7 +72,7 @@ def run():index_process = Process(target=run_indexing_continuously)index_process.start()-+# Track index process crashes - list of (timestamp, exit_code, pid) tuplesindex_crash_history: list[tuple[datetime, int, int]] = []@@ -80,25 +81,25 @@ def run():crash_time = datetime.now()exit_code = index_process.exitcode or -1pid = index_process.pid-+# Record this crashindex_crash_history.append((crash_time, exit_code, pid))-+# Remove crashes older than 1 hourone_hour_ago = crash_time - timedelta(hours=1)index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]-+# Check if we've exceeded the crash thresholdif len(index_crash_history) > 5:crash_details = []for crash_time, exit_code, pid in index_crash_history:crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")-+error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))logger.error(error_msg)raise RuntimeError(error_msg)-+logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "f"Crash count in last hour: {len(index_crash_history)}")index_process = Process(target=run_indexing_continuously)@@ -127,20 +128,20 @@ def process_batch_continuously():def process_batch():"""Process a single batch of URLs by crawling them sequentially with rate limiting.-+This function handles the core crawling workflow:1. Gets a batch of URLs from the Redis URL queue2. Crawls each URL sequentially with configurable delay between requests3. Records crawl results in the database for URL tracking4. Pushes the completed batch to Redis queue for indexing-+The sequential crawling with delays respects rate limits and reduces load on target servers.Each batch is processed as a HashedBatch object containing metadata and crawl results."""user_id = "test"urls = url_queue.get_batch(user_id)logger.info(f"Processing batch of {len(urls)} URLs")-+# Process URLs sequentially with rate limitingresults = []for i, url in enumerate(urls):@@ -148,7 +149,7 @@ def process_batch():# Add delay with 10% random fuzzdelay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())time.sleep(delay)-+result = crawl_url(url)results.append(result)logger.debug("Result", result)@@ -179,17 +180,17 @@ def run_indexing_continuously():def run_indexing():"""Process completed crawl batches and integrate results into the search index.-+This function handles the indexing workflow:1. Pulls completed crawl batches from Redis queue (up to 10 at once)2. Indexes batches locally using the tiny search engine indexer3. For top terms, syncs high-scoring local results with the remote Mwmbl index4. Downloads updated remote results and merges them back into local index-+The sync process ensures that high-quality local crawl results get submittedto the main Mwmbl search index, while also keeping the local index updatedwith the latest remote results for better search quality.-+Only results that score higher than existing remote results are submitted,preventing low-quality content from polluting the main index."""@@ -226,7 +227,7 @@ def run_indexing():if new_high_score:result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,score=doc.score, term=doc.term, state=doc.state) for doc in new_items]- results = Results(api_key=API_KEY, results=result_items)+ results = Results(api_key=MWMBL_API_KEY, results=result_items)logger.info(f"Posting {len(result_items)} results")response = requests.post("https://mwmbl.org/api/v1/crawler/results", json=results.dict()diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pyindex 914f938..e010914 100644--- a/mwmbl/crawler/env_vars.py+++ b/mwmbl/crawler/env_vars.py@@ -1,29 +1,19 @@"""Environment variables configuration for the crawler module."""-import os--# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned-REASSIGN_MIN_HOURS = int(os.environ.get("REASSIGN_MIN_HOURS", "5"))-BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))-MAX_URLS_PER_TOP_DOMAIN = int(os.environ.get("MAX_URLS_PER_TOP_DOMAIN", "100"))-MAX_TOP_DOMAINS = int(os.environ.get("MAX_TOP_DOMAINS", "500"))-MAX_OTHER_DOMAINS = int(os.environ.get("MAX_OTHER_DOMAINS", "50000"))+import os-# Retrieval configuration-TIMEOUT_SECONDS = int(os.environ.get("TIMEOUT_SECONDS", "3"))-MAX_FETCH_SIZE = int(os.environ.get("MAX_FETCH_SIZE", str(1024 * 1024)))-MAX_NEW_LINKS = int(os.environ.get("MAX_NEW_LINKS", "50"))-MAX_EXTRA_LINKS = int(os.environ.get("MAX_EXTRA_LINKS", "50"))-MAX_SITE_URLS = int(os.environ.get("MAX_SITE_URLS", "100"))# Crawler worker configuration-CRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))-CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))+CRAWLER_WORKERS = int(os.environ.get("CRAWLER_WORKERS", "10"))+CRAWL_THREADS = int(os.environ.get("CRAWL_THREADS", "20"))# Rate limiting configuration-CRAWL_DELAY_SECONDS = float(os.environ.get('CRAWL_DELAY_SECONDS', '0.0'))+CRAWL_DELAY_SECONDS = float(os.environ.get("CRAWL_DELAY_SECONDS", "0.0"))# Redis configuration-REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')+REDIS_URL = os.environ.get("REDIS_URL", "redis://127.0.0.1:6379")++# API configuration+MWMBL_API_KEY = os.environ["MWMBL_API_KEY"]diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 8640c14..4217bdb 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -13,7 +13,6 @@ from urllib3.exceptions import NewConnectionError, MaxRetryErrorfrom mwmbl.justext.core import html_to_domfrom mwmbl.justext.paragraph import Paragraph-from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLSVERSION = "2.0"@@ -23,8 +22,12 @@ ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,POST_BATCH_URL = '/api/v1/crawler/batches/'POST_NEW_BATCH_URL = '/api/v1/crawler/batches/new'+TIMEOUT_SECONDS = 3+MAX_FETCH_SIZE = 1024*1024MAX_URL_LENGTH = 150BAD_URL_REGEX = re.compile(r'\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$')+MAX_NEW_LINKS = 50+MAX_EXTRA_LINKS = 50NUM_TITLE_CHARS = 65NUM_EXTRACT_CHARS = 155DEFAULT_ENCODING = 'utf8'