An open source, non-profit web search engine
Make URLDatabase a context manager
Daoud Clarkeb8ea3066d4d7
diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex c80c7a9..d3f0983 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -1,12 +1,9 @@-import os-import shutilfrom multiprocessing import Process, Queuefrom pathlib import Pathfrom django.apps import AppConfigfrom django.conf import settings-from mwmbl.crawler.urls import URLDatabasefrom mwmbl.database import Databasefrom mwmbl.indexer.indexdb import IndexDatabase@@ -36,7 +33,6 @@ class MwmblConfig(AppConfig):page_size=PAGE_SIZE)with Database() as db:- url_db = URLDatabase(db.connection)index_db = IndexDatabase(db.connection)index_db.create_tables()diff --git a/mwmbl/background.py b/mwmbl/background.pyindex c7d2b3b..d51be4a 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -7,13 +7,10 @@ from logging import getLogger, basicConfigfrom pathlib import Pathfrom time import sleep-from mwmbl.crawler.urls import URLDatabase-from mwmbl.database import Databasefrom mwmbl.indexer import index_batches, historicalfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAME-basicConfig(stream=sys.stdout, level=logging.INFO)logger = getLogger(__name__)@@ -21,9 +18,6 @@ logger = getLogger(__name__)def run(data_path: str):logger.info("Started background process")- with Database() as db:- url_db = URLDatabase(db.connection)-historical.run()index_path = Path(data_path) / INDEX_NAMEbatch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 2edd4d8..692db79 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -15,7 +15,6 @@ from redis import Redisfrom mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatchfrom mwmbl.crawler.stats import MwmblStats, StatsManager-from mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatusfrom mwmbl.database import Databasefrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatusdiff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex b50b660..d0154f4 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -51,12 +51,18 @@ class FoundURL:class URLDatabase:- def __init__(self, connection):+ def __init__(self):+ self.urls = None++ def __enter__(self):try:self.urls = BloomFilter.open(settings.URLS_BLOOM_FILTER_PATH)except FileNotFoundError:logger.info("No existing bloom filter found, creating a new one")- self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 0.01, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)+ self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)++ def __exit__(self, exc_type, exc_val, exc_tb):+ self.urls.close()def update_found_urls(self, found_urls: list[FoundURL]):"""diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 10971df..329b469 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -5,18 +5,14 @@ from collections import defaultdictfrom logging import getLoggerfrom typing import Collection, Iterable-import spacy-from mwmbl.indexer import process_batch-from spacy import Language-from mwmbl.crawler.batch import HashedBatch, Item-from mwmbl.crawler.urls import URLDatabase, URLStatus-from mwmbl.database import Database+from mwmbl.crawler.urls import URLStatus+from mwmbl.indexer import process_batchfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.indexer.indexdb import BatchStatusfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex-from mwmbl.utils import add_term_info, add_term_infos+from mwmbl.utils import add_term_infoslogger = getLogger(__name__)diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex f8cd6b7..6163974 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -44,8 +44,8 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qublacklist_retrieval_time = datetime.now() - startlogger.info(f"Recording URLs in database for {len(batches)} batches, with {len(blacklist_domains)} blacklist "f"domains, retrieved in {blacklist_retrieval_time.total_seconds()} seconds")- with Database() as db:- url_db = URLDatabase(db.connection)++ with URLDatabase() as url_db:url_scores = defaultdict(float)url_users = {}url_timestamps = {}