An open source, non-profit web search engine
Add a script to update URLs when running locally so we can profile the code
Daoud Clarkea5e688eeec26
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex a383f03..99ac075 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -1,3 +1,4 @@+import loggingfrom collections import defaultdictfrom datetime import datetime, timezone, timedeltafrom logging import getLogger@@ -7,6 +8,7 @@ from typing import Collectionfrom urllib.parse import urlparseimport requests+from redis import Redisfrom mwmbl.crawler.batch import HashedBatchfrom mwmbl.crawler.domains import DomainLinkDatabase@@ -34,8 +36,8 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):sleep(10)-def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue):- process_batch.run(batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, 1000,+def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue, num_batches: int = 1000):+ process_batch.run(batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, num_batches,new_item_queue)@@ -131,3 +133,11 @@ def process_link(user_id_hash, crawled_page_domain, link, timestamp, url_timestadef get_datetime_from_timestamp(timestamp: float) -> datetime:batch_datetime = datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=timestamp)return batch_datetime+++if __name__ == "__main__":+ logging.basicConfig(level=logging.INFO)+ redis: Redis = Redis.from_url("redis://127.0.0.1:6379", decode_responses=True)+ batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)+ url_queue = RedisURLQueue(redis, lambda: set())+ run(batch_cache, url_queue, num_batches=100)diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex c35b626..d8cfc44 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -12,7 +12,7 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.rank import HeuristicRanker-def get_curated_domains():+def get_curated_domains() -> set[str]:curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))return curated_domains