An open source, non-profit web search engine
Move more paths to settings
Daoud Clarkee83751894a36
diff --git a/analyse/index_concurrency.py b/analyse/index_concurrency.pyindex 3084101..25d4cac 100644--- a/analyse/index_concurrency.py+++ b/analyse/index_concurrency.py@@ -5,7 +5,6 @@ from multiprocessing.pool import ThreadPoolfrom pathlib import Pathfrom random import Random-from mwmbl.indexer.paths import INDEX_NAMEfrom mwmbl.tinysearchengine.indexer import TinyIndex, Documentrandom = Random(2)@@ -25,7 +24,7 @@ PAGES = [random.randint(0, 10240000) for _ in range(20)]NUM_PAGES = 10240000PAGE_SIZE = 4096-INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / INDEX_NAME+INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / "index-v2.tinysearch"# INDEX_PATH = 'temp-index.tinysearch'diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex fe29c69..0b867c9 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -14,10 +14,9 @@ class MwmblConfig(AppConfig):def ready(self):# Imports here to avoid AppRegistryNotReady exception- from mwmbl.indexer.paths import INDEX_NAMEfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE- index_path = Path(settings.DATA_PATH) / INDEX_NAME+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtry:existing_index = TinyIndex(item_factory=Document, index_path=index_path)if existing_index.page_size != PAGE_SIZE or existing_index.num_pages != settings.NUM_PAGES:diff --git a/mwmbl/background.py b/mwmbl/background.pyindex fdcd82b..b8ebd8e 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -12,10 +12,8 @@ from django.conf import settingsfrom mwmbl.indexer import index_batches, historicalfrom mwmbl.indexer.batch_cache import BatchCache-from mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAMEfrom mwmbl.models import OldIndexfrom mwmbl.tinysearchengine.copy_index import copy_pages-from mwmbl.tinysearchengine.indexer import TinyIndex, DocumentNUM_PAGES_TO_COPY = 1024@@ -28,8 +26,8 @@ def run(data_path: str):logger.info("Started background process")historical.run()- index_path = Path(data_path) / INDEX_NAME- batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)+ index_path = Path(data_path) / settings.INDEX_NAME+ batch_cache = BatchCache(Path(data_path) / settings.BATCH_DIR_NAME)while True:try:@@ -68,7 +66,7 @@ def copy_all_indexes(new_index_path):def copy_indexes_continuously():- new_index_path = Path(settings.DATA_PATH) / INDEX_NAME+ new_index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEwhile True:num_updated = 0try:diff --git a/mwmbl/indexer/paths.py b/mwmbl/indexer/paths.pyindex c996c57..5b02c41 100644--- a/mwmbl/indexer/paths.py+++ b/mwmbl/indexer/paths.py@@ -26,6 +26,3 @@ TOP_DOMAINS_JSON_PATH = TINYSEARCH_DATA_DIR / 'hn-top-domains.json'MWMBL_DATA_DIR = DATA_DIR / "mwmbl"CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/*/*/*/*.json.gz"LINK_COUNT_PATH = MWMBL_DATA_DIR / 'crawl-counts.json'--INDEX_NAME = 'index-v2.tinysearch'-BATCH_DIR_NAME = 'batches'\ No newline at end of filediff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex d3e8771..e98ea4b 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -1,7 +1,6 @@from collections import defaultdictfrom datetime import datetime, timezone, timedeltafrom logging import getLogger-from multiprocessing import Queuefrom pathlib import Pathfrom time import sleepfrom typing import Collection@@ -10,24 +9,21 @@ from urllib.parse import urlparsefrom mwmbl.crawler.batch import HashedBatchfrom mwmbl.crawler.domains import DomainLinkDatabasefrom mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL-from mwmbl.database import Database-from mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer import process_batchfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklistedfrom mwmbl.indexer.index_batches import get_url_error_statusfrom mwmbl.indexer.indexdb import BatchStatus-from mwmbl.indexer.paths import BATCH_DIR_NAMEfrom mwmbl.redis_url_queue import RedisURLQueue-from mwmbl.settings import UNKNOWN_DOMAIN_MULTIPLIER, SCORE_FOR_SAME_DOMAIN, \- SCORE_FOR_DIFFERENT_DOMAIN, SCORE_FOR_ROOT_PATH, EXTRA_LINK_MULTIPLIERfrom mwmbl.utils import get_domain+from django.conf import settings+logger = getLogger(__name__)def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):- batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)+ batch_cache = BatchCache(Path(data_path) / settings.BATCH_DIR_NAME)while True:try:run(batch_cache, new_item_queue)diff --git a/mwmbl/main.py b/mwmbl/main.pyindex c287786..b01c18d 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -1,12 +1,10 @@import os-from pathlib import Pathimport djangoimport uvicornfrom django.core.management import call_commandfrom redis import Redis-from mwmbl.indexer.paths import INDEX_NAMEfrom mwmbl.indexer.update_urls import update_urls_continuouslyfrom mwmbl.redis_url_queue import RedisURLQueuediff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex c1a37ad..f65ab2b 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -5,7 +5,6 @@ from django.conf import settingsfrom redis import Redisfrom mwmbl.indexer.batch_cache import BatchCache-from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAMEfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -14,9 +13,9 @@ from mwmbl.tinysearchengine.rank import HeuristicRankerredis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)queued_batches = RedisURLQueue(redis)completer = Completer()-index_path = Path(settings.DATA_PATH) / INDEX_NAME+index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()ranker = HeuristicRanker(tiny_index, completer)-batch_cache = BatchCache(Path(settings.DATA_PATH) / BATCH_DIR_NAME)+batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex d1ca583..27a0a63 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -168,3 +168,5 @@ FOOTER_LINKS = []++BATCH_DIR_NAME = 'batches'diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex 0c8f444..a795754 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -26,6 +26,8 @@ ALLOWED_HOSTS = ["localhost", "127.0.0.1"]EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'DATA_PATH = "./devdata"+INDEX_NAME = 'index-v2.tinysearch'+NUM_PAGES = 2560URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex 35a7f38..7a82d23 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -30,7 +30,10 @@ EMAIL_USE_TLS = TrueDATA_PATH = "/app/storage"-NUM_PAGES = 10240000+INDEX_NAME = 'index-v2-400G.tinysearch'++# 400GB index+NUM_PAGES = 102400000URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 200_000_000diff --git a/test/test_copy_index.py b/test/test_copy_index.pyindex e7b5152..25f7e31 100644--- a/test/test_copy_index.py+++ b/test/test_copy_index.py@@ -1,20 +1,21 @@from pathlib import Pathfrom tempfile import TemporaryDirectory-from mwmbl.indexer.paths import INDEX_NAMEfrom mwmbl.tinysearchengine.copy_index import copy_pagesfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE+from django.conf import settings+NUM_PAGES = 10000def test_copy_pages():search_term = "apple"- old_index_path = Path(__file__).parent.parent / "devdata" / INDEX_NAME+ old_index_path = Path(__file__).parent.parent / "devdata" / settings.INDEX_NAME# Make a temporary directory for the new indexwith TemporaryDirectory() as new_index_dir:- new_index_path = str(Path(new_index_dir) / INDEX_NAME)+ new_index_path = str(Path(new_index_dir) / settings.INDEX_NAME)TinyIndex.create(Document, new_index_path, NUM_PAGES, PAGE_SIZE)with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:start_index = old_index.get_key_page_index(search_term)@@ -24,4 +25,9 @@ def test_copy_pages():new_items = new_index.retrieve(search_term)assert len(old_items) > 0- assert old_items == new_items++ old_urls = {item.url for item in old_items}+ new_urls = {item.url for item in new_items}++ # The scores may change, so we only compare the URLs+ assert old_urls == new_urls