An open source, non-profit web search engine
Speed up
Daoud Clarkef9e324ad43fe
diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.pyindex 471f430..aa5bfc9 100644--- a/mwmbl/count_urls.py+++ b/mwmbl/count_urls.py@@ -7,9 +7,11 @@ from django.conf import settingsfrom redis import Redisfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.utils import batchINDEX_URL_COUNT_KEY = "index-count-{date}"LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30+NUM_PAGES_IN_BATCH = 1024logger = getLogger(__name__)@@ -25,12 +27,15 @@ def count_urls() -> int:logger.info(f"Counting URLs in index {index_path}")- key = INDEX_URL_COUNT_KEY.format(date.today())+ key = INDEX_URL_COUNT_KEY.format(date=date.today())with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:# Count using a Redis hyperloglog to avoid memory issues.- for page in range(tiny_index.num_pages):- for doc in tiny_index.get_page(page):- redis.pfadd(key, doc.url)+ for page_batch in batch(range(tiny_index.num_pages), NUM_PAGES_IN_BATCH):+ urls = set()+ for page in page_batch:+ urls |= {doc.url for doc in tiny_index.get_page(page)}+ redis.pfadd(key, *urls)+ logger.info(f"Counted {page} pages of {tiny_index.num_pages}.")count = redis.pfcount(key)redis.expire(key, LONG_EXPIRE_SECONDS)@@ -43,3 +48,11 @@ def get_url_count(date_: date) -> int:redis = get_redis()key = INDEX_URL_COUNT_KEY.format(date_)return redis.pfcount(key)+++if __name__ == "__main__":+ # configure logging+ import logging+ logging.basicConfig(level=logging.INFO)++ count_urls()diff --git a/mwmbl/main.py b/mwmbl/main.pyindex b01c18d..c7e3003 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -5,6 +5,7 @@ import uvicornfrom django.core.management import call_commandfrom redis import Redis+from mwmbl.count_urls import count_urlsfrom mwmbl.indexer.update_urls import update_urls_continuouslyfrom mwmbl.redis_url_queue import RedisURLQueue@@ -29,6 +30,8 @@ def run():background.run(settings.DATA_PATH)elif mwmbl_app == "copy_indexes":background.copy_indexes_continuously()+ elif mwmbl_app == "count_urls":+ count_urls()elif mwmbl_app == "server":uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000)else:diff --git a/mwmbl/utils.py b/mwmbl/utils.pyindex 12cc051..97be262 100644--- a/mwmbl/utils.py+++ b/mwmbl/utils.py@@ -1,4 +1,5 @@import re+from typing import Sequencefrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex@@ -6,7 +7,7 @@ from mwmbl.tinysearchengine.indexer import Document, TinyIndexDOMAIN_REGEX = re.compile(r".*://([^/]*)")-def batch(items: list, batch_size):+def batch(items: Sequence, batch_size):"""Adapted from https://stackoverflow.com/a/8290508"""