Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Speed up

Daoud Clarkef9e324ad43fe
diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.py
index 471f430..aa5bfc9 100644
--- a/mwmbl/count_urls.py
+++ b/mwmbl/count_urls.py
@@ -7,9 +7,11 @@ from django.conf import settings
from redis import Redis
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.utils import batch
INDEX_URL_COUNT_KEY = "index-count-{date}"
LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30
+NUM_PAGES_IN_BATCH = 1024
logger = getLogger(__name__)
@@ -25,12 +27,15 @@ def count_urls() -> int:
logger.info(f"Counting URLs in index {index_path}")
- key = INDEX_URL_COUNT_KEY.format(date.today())
+ key = INDEX_URL_COUNT_KEY.format(date=date.today())
with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:
# Count using a Redis hyperloglog to avoid memory issues.
- for page in range(tiny_index.num_pages):
- for doc in tiny_index.get_page(page):
- redis.pfadd(key, doc.url)
+ for page_batch in batch(range(tiny_index.num_pages), NUM_PAGES_IN_BATCH):
+ urls = set()
+ for page in page_batch:
+ urls |= {doc.url for doc in tiny_index.get_page(page)}
+ redis.pfadd(key, *urls)
+ logger.info(f"Counted {page} pages of {tiny_index.num_pages}.")
count = redis.pfcount(key)
redis.expire(key, LONG_EXPIRE_SECONDS)
@@ -43,3 +48,11 @@ def get_url_count(date_: date) -> int:
redis = get_redis()
key = INDEX_URL_COUNT_KEY.format(date_)
return redis.pfcount(key)
+
+
+if __name__ == "__main__":
+ # configure logging
+ import logging
+ logging.basicConfig(level=logging.INFO)
+
+ count_urls()
diff --git a/mwmbl/main.py b/mwmbl/main.py
index b01c18d..c7e3003 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -5,6 +5,7 @@ import uvicorn
from django.core.management import call_command
from redis import Redis
+from mwmbl.count_urls import count_urls
from mwmbl.indexer.update_urls import update_urls_continuously
from mwmbl.redis_url_queue import RedisURLQueue
@@ -29,6 +30,8 @@ def run():
background.run(settings.DATA_PATH)
elif mwmbl_app == "copy_indexes":
background.copy_indexes_continuously()
+ elif mwmbl_app == "count_urls":
+ count_urls()
elif mwmbl_app == "server":
uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000)
else:
diff --git a/mwmbl/utils.py b/mwmbl/utils.py
index 12cc051..97be262 100644
--- a/mwmbl/utils.py
+++ b/mwmbl/utils.py
@@ -1,4 +1,5 @@
import re
+from typing import Sequence
from mwmbl.indexer.index import tokenize_document
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
@@ -6,7 +7,7 @@ from mwmbl.tinysearchengine.indexer import Document, TinyIndex
DOMAIN_REGEX = re.compile(r".*://([^/]*)")
-def batch(items: list, batch_size):
+def batch(items: Sequence, batch_size):
"""
Adapted from https://stackoverflow.com/a/8290508
"""