Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Move more paths to settings

Daoud Clarkee83751894a36
diff --git a/analyse/index_concurrency.py b/analyse/index_concurrency.py
index 3084101..25d4cac 100644
--- a/analyse/index_concurrency.py
+++ b/analyse/index_concurrency.py
@@ -5,7 +5,6 @@ from multiprocessing.pool import ThreadPool
from pathlib import Path
from random import Random
-from mwmbl.indexer.paths import INDEX_NAME
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
random = Random(2)
@@ -25,7 +24,7 @@ PAGES = [random.randint(0, 10240000) for _ in range(20)]
NUM_PAGES = 10240000
PAGE_SIZE = 4096
-INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / INDEX_NAME
+INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / "index-v2.tinysearch"
# INDEX_PATH = 'temp-index.tinysearch'
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index fe29c69..0b867c9 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -14,10 +14,9 @@ class MwmblConfig(AppConfig):
def ready(self):
# Imports here to avoid AppRegistryNotReady exception
- from mwmbl.indexer.paths import INDEX_NAME
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
- index_path = Path(settings.DATA_PATH) / INDEX_NAME
+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
try:
existing_index = TinyIndex(item_factory=Document, index_path=index_path)
if existing_index.page_size != PAGE_SIZE or existing_index.num_pages != settings.NUM_PAGES:
diff --git a/mwmbl/background.py b/mwmbl/background.py
index fdcd82b..b8ebd8e 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -12,10 +12,8 @@ from django.conf import settings
from mwmbl.indexer import index_batches, historical
from mwmbl.indexer.batch_cache import BatchCache
-from mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAME
from mwmbl.models import OldIndex
from mwmbl.tinysearchengine.copy_index import copy_pages
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document
NUM_PAGES_TO_COPY = 1024
@@ -28,8 +26,8 @@ def run(data_path: str):
logger.info("Started background process")
historical.run()
- index_path = Path(data_path) / INDEX_NAME
- batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)
+ index_path = Path(data_path) / settings.INDEX_NAME
+ batch_cache = BatchCache(Path(data_path) / settings.BATCH_DIR_NAME)
while True:
try:
@@ -68,7 +66,7 @@ def copy_all_indexes(new_index_path):
def copy_indexes_continuously():
- new_index_path = Path(settings.DATA_PATH) / INDEX_NAME
+ new_index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
while True:
num_updated = 0
try:
diff --git a/mwmbl/indexer/paths.py b/mwmbl/indexer/paths.py
index c996c57..5b02c41 100644
--- a/mwmbl/indexer/paths.py
+++ b/mwmbl/indexer/paths.py
@@ -26,6 +26,3 @@ TOP_DOMAINS_JSON_PATH = TINYSEARCH_DATA_DIR / 'hn-top-domains.json'
MWMBL_DATA_DIR = DATA_DIR / "mwmbl"
CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/*/*/*/*.json.gz"
LINK_COUNT_PATH = MWMBL_DATA_DIR / 'crawl-counts.json'
-
-INDEX_NAME = 'index-v2.tinysearch'
-BATCH_DIR_NAME = 'batches'
\ No newline at end of file
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index d3e8771..e98ea4b 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -1,7 +1,6 @@
from collections import defaultdict
from datetime import datetime, timezone, timedelta
from logging import getLogger
-from multiprocessing import Queue
from pathlib import Path
from time import sleep
from typing import Collection
@@ -10,24 +9,21 @@ from urllib.parse import urlparse
from mwmbl.crawler.batch import HashedBatch
from mwmbl.crawler.domains import DomainLinkDatabase
from mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL
-from mwmbl.database import Database
-from mwmbl.hn_top_domains_filtered import DOMAINS
from mwmbl.indexer import process_batch
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklisted
from mwmbl.indexer.index_batches import get_url_error_status
from mwmbl.indexer.indexdb import BatchStatus
-from mwmbl.indexer.paths import BATCH_DIR_NAME
from mwmbl.redis_url_queue import RedisURLQueue
-from mwmbl.settings import UNKNOWN_DOMAIN_MULTIPLIER, SCORE_FOR_SAME_DOMAIN, \
- SCORE_FOR_DIFFERENT_DOMAIN, SCORE_FOR_ROOT_PATH, EXTRA_LINK_MULTIPLIER
from mwmbl.utils import get_domain
+from django.conf import settings
+
logger = getLogger(__name__)
def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):
- batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)
+ batch_cache = BatchCache(Path(data_path) / settings.BATCH_DIR_NAME)
while True:
try:
run(batch_cache, new_item_queue)
diff --git a/mwmbl/main.py b/mwmbl/main.py
index c287786..b01c18d 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -1,12 +1,10 @@
import os
-from pathlib import Path
import django
import uvicorn
from django.core.management import call_command
from redis import Redis
-from mwmbl.indexer.paths import INDEX_NAME
from mwmbl.indexer.update_urls import update_urls_continuously
from mwmbl.redis_url_queue import RedisURLQueue
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index c1a37ad..f65ab2b 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -5,7 +5,6 @@ from django.conf import settings
from redis import Redis
from mwmbl.indexer.batch_cache import BatchCache
-from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -14,9 +13,9 @@ from mwmbl.tinysearchengine.rank import HeuristicRanker
redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)
queued_batches = RedisURLQueue(redis)
completer = Completer()
-index_path = Path(settings.DATA_PATH) / INDEX_NAME
+index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
ranker = HeuristicRanker(tiny_index, completer)
-batch_cache = BatchCache(Path(settings.DATA_PATH) / BATCH_DIR_NAME)
+batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index d1ca583..27a0a63 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -168,3 +168,5 @@ FOOTER_LINKS = [
]
+
+BATCH_DIR_NAME = 'batches'
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index 0c8f444..a795754 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -26,6 +26,8 @@ ALLOWED_HOSTS = ["localhost", "127.0.0.1"]
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
DATA_PATH = "./devdata"
+INDEX_NAME = 'index-v2.tinysearch'
+
NUM_PAGES = 2560
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.py
index 35a7f38..7a82d23 100644
--- a/mwmbl/settings_prod.py
+++ b/mwmbl/settings_prod.py
@@ -30,7 +30,10 @@ EMAIL_USE_TLS = True
DATA_PATH = "/app/storage"
-NUM_PAGES = 10240000
+INDEX_NAME = 'index-v2-400G.tinysearch'
+
+# 400GB index
+NUM_PAGES = 102400000
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 200_000_000
diff --git a/test/test_copy_index.py b/test/test_copy_index.py
index e7b5152..25f7e31 100644
--- a/test/test_copy_index.py
+++ b/test/test_copy_index.py
@@ -1,20 +1,21 @@
from pathlib import Path
from tempfile import TemporaryDirectory
-from mwmbl.indexer.paths import INDEX_NAME
from mwmbl.tinysearchengine.copy_index import copy_pages
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
+from django.conf import settings
+
NUM_PAGES = 10000
def test_copy_pages():
search_term = "apple"
- old_index_path = Path(__file__).parent.parent / "devdata" / INDEX_NAME
+ old_index_path = Path(__file__).parent.parent / "devdata" / settings.INDEX_NAME
# Make a temporary directory for the new index
with TemporaryDirectory() as new_index_dir:
- new_index_path = str(Path(new_index_dir) / INDEX_NAME)
+ new_index_path = str(Path(new_index_dir) / settings.INDEX_NAME)
TinyIndex.create(Document, new_index_path, NUM_PAGES, PAGE_SIZE)
with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:
start_index = old_index.get_key_page_index(search_term)
@@ -24,4 +25,9 @@ def test_copy_pages():
new_items = new_index.retrieve(search_term)
assert len(old_items) > 0
- assert old_items == new_items
+
+ old_urls = {item.url for item in old_items}
+ new_urls = {item.url for item in new_items}
+
+ # The scores may change, so we only compare the URLs
+ assert old_urls == new_urls