Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

fix: botched merge forgot some lines

thiswillbeyourgithubea1c6f8fecff
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index cd5f4c3..5f316b8 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -11,9 +11,7 @@ import requests
from django.conf import settings
from redis import Redis
-CRAWLER_VERSION: str = "0.2.0"
-
-from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_THREADS, CRAWL_DELAY_SECONDS
+from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_DELAY_SECONDS, MWMBL_API_KEY, REDIS_URL
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -30,14 +28,17 @@ print("Mwmbl crawling statistics: https://mwmbl.org/stats")
django.setup()
-
from mwmbl.indexer.update_urls import record_urls_in_database
from mwmbl.crawler.retrieve import crawl_batch, crawl_url
from mwmbl.crawler.batch import HashedBatch, Result, Results
from mwmbl.indexer.index_batches import index_batches, index_pages
-API_KEY = MWMBL_API_KEY
+logger = logging.getLogger(__name__)
+FORMAT = "%(process)d:%(levelname)s:%(name)s:%(message)s"
+logging.basicConfig(level=logging.INFO, format=FORMAT)
+
BATCH_QUEUE_KEY = "batch-queue"
+CRAWLER_VERSION: str = "0.2.0"
redis = Redis.from_url(
@@ -71,7 +72,7 @@ def run():
index_process = Process(target=run_indexing_continuously)
index_process.start()
-
+
# Track index process crashes - list of (timestamp, exit_code, pid) tuples
index_crash_history: list[tuple[datetime, int, int]] = []
@@ -80,25 +81,25 @@ def run():
crash_time = datetime.now()
exit_code = index_process.exitcode or -1
pid = index_process.pid
-
+
# Record this crash
index_crash_history.append((crash_time, exit_code, pid))
-
+
# Remove crashes older than 1 hour
one_hour_ago = crash_time - timedelta(hours=1)
index_crash_history = [(t, c, p) for t, c, p in index_crash_history if t > one_hour_ago]
-
+
# Check if we've exceeded the crash threshold
if len(index_crash_history) > 5:
crash_details = []
for crash_time, exit_code, pid in index_crash_history:
crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")
-
+
error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "
f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))
logger.error(error_msg)
raise RuntimeError(error_msg)
-
+
logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "
f"Crash count in last hour: {len(index_crash_history)}")
index_process = Process(target=run_indexing_continuously)
@@ -127,20 +128,20 @@ def process_batch_continuously():
def process_batch():
"""
Process a single batch of URLs by crawling them sequentially with rate limiting.
-
+
This function handles the core crawling workflow:
1. Gets a batch of URLs from the Redis URL queue
2. Crawls each URL sequentially with configurable delay between requests
3. Records crawl results in the database for URL tracking
4. Pushes the completed batch to Redis queue for indexing
-
+
The sequential crawling with delays respects rate limits and reduces load on target servers.
Each batch is processed as a HashedBatch object containing metadata and crawl results.
"""
user_id = "test"
urls = url_queue.get_batch(user_id)
logger.info(f"Processing batch of {len(urls)} URLs")
-
+
# Process URLs sequentially with rate limiting
results = []
for i, url in enumerate(urls):
@@ -148,7 +149,7 @@ def process_batch():
# Add delay with 10% random fuzz
delay = CRAWL_DELAY_SECONDS * (0.9 + 0.2 * random.random())
time.sleep(delay)
-
+
result = crawl_url(url)
results.append(result)
logger.debug("Result", result)
@@ -179,17 +180,17 @@ def run_indexing_continuously():
def run_indexing():
"""
Process completed crawl batches and integrate results into the search index.
-
+
This function handles the indexing workflow:
1. Pulls completed crawl batches from Redis queue (up to 10 at once)
2. Indexes batches locally using the tiny search engine indexer
3. For top terms, syncs high-scoring local results with the remote Mwmbl index
4. Downloads updated remote results and merges them back into local index
-
+
The sync process ensures that high-quality local crawl results get submitted
to the main Mwmbl search index, while also keeping the local index updated
with the latest remote results for better search quality.
-
+
Only results that score higher than existing remote results are submitted,
preventing low-quality content from polluting the main index.
"""
@@ -226,7 +227,7 @@ def run_indexing():
if new_high_score:
result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
- results = Results(api_key=API_KEY, results=result_items)
+ results = Results(api_key=MWMBL_API_KEY, results=result_items)
logger.info(f"Posting {len(result_items)} results")
response = requests.post(
"https://mwmbl.org/api/v1/crawler/results", json=results.dict()
diff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.py
index 914f938..e010914 100644
--- a/mwmbl/crawler/env_vars.py
+++ b/mwmbl/crawler/env_vars.py
@@ -1,29 +1,19 @@
"""
Environment variables configuration for the crawler module.
"""
-import os
-
-# Client has one hour to crawl a URL that has been assigned to them, or it will be reassigned
-REASSIGN_MIN_HOURS = int(os.environ.get("REASSIGN_MIN_HOURS", "5"))
-BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))
-MAX_URLS_PER_TOP_DOMAIN = int(os.environ.get("MAX_URLS_PER_TOP_DOMAIN", "100"))
-MAX_TOP_DOMAINS = int(os.environ.get("MAX_TOP_DOMAINS", "500"))
-MAX_OTHER_DOMAINS = int(os.environ.get("MAX_OTHER_DOMAINS", "50000"))
+import os
-# Retrieval configuration
-TIMEOUT_SECONDS = int(os.environ.get("TIMEOUT_SECONDS", "3"))
-MAX_FETCH_SIZE = int(os.environ.get("MAX_FETCH_SIZE", str(1024 * 1024)))
-MAX_NEW_LINKS = int(os.environ.get("MAX_NEW_LINKS", "50"))
-MAX_EXTRA_LINKS = int(os.environ.get("MAX_EXTRA_LINKS", "50"))
-MAX_SITE_URLS = int(os.environ.get("MAX_SITE_URLS", "100"))
# Crawler worker configuration
-CRAWLER_WORKERS = int(os.environ.get('CRAWLER_WORKERS', '10'))
-CRAWL_THREADS = int(os.environ.get('CRAWL_THREADS', '20'))
+CRAWLER_WORKERS = int(os.environ.get("CRAWLER_WORKERS", "10"))
+CRAWL_THREADS = int(os.environ.get("CRAWL_THREADS", "20"))
# Rate limiting configuration
-CRAWL_DELAY_SECONDS = float(os.environ.get('CRAWL_DELAY_SECONDS', '0.0'))
+CRAWL_DELAY_SECONDS = float(os.environ.get("CRAWL_DELAY_SECONDS", "0.0"))
# Redis configuration
-REDIS_URL = os.environ.get('REDIS_URL', 'redis://127.0.0.1:6379')
+REDIS_URL = os.environ.get("REDIS_URL", "redis://127.0.0.1:6379")
+
+# API configuration
+MWMBL_API_KEY = os.environ["MWMBL_API_KEY"]
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 8640c14..4217bdb 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -13,7 +13,6 @@ from urllib3.exceptions import NewConnectionError, MaxRetryError
from mwmbl.justext.core import html_to_dom
from mwmbl.justext.paragraph import Paragraph
-from mwmbl.crawler.env_vars import TIMEOUT_SECONDS, MAX_FETCH_SIZE, MAX_NEW_LINKS, MAX_EXTRA_LINKS, MAX_SITE_URLS
VERSION = "2.0"
@@ -23,8 +22,12 @@ ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
POST_BATCH_URL = '/api/v1/crawler/batches/'
POST_NEW_BATCH_URL = '/api/v1/crawler/batches/new'
+TIMEOUT_SECONDS = 3
+MAX_FETCH_SIZE = 1024*1024
MAX_URL_LENGTH = 150
BAD_URL_REGEX = re.compile(r'\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$')
+MAX_NEW_LINKS = 50
+MAX_EXTRA_LINKS = 50
NUM_TITLE_CHARS = 65
NUM_EXTRACT_CHARS = 155
DEFAULT_ENCODING = 'utf8'