Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Make URLDatabase a context manager

Daoud Clarkeb8ea3066d4d7
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index c80c7a9..d3f0983 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -1,12 +1,9 @@
-import os
-import shutil
from multiprocessing import Process, Queue
from pathlib import Path
from django.apps import AppConfig
from django.conf import settings
-from mwmbl.crawler.urls import URLDatabase
from mwmbl.database import Database
from mwmbl.indexer.indexdb import IndexDatabase
@@ -36,7 +33,6 @@ class MwmblConfig(AppConfig):
page_size=PAGE_SIZE)
with Database() as db:
- url_db = URLDatabase(db.connection)
index_db = IndexDatabase(db.connection)
index_db.create_tables()
diff --git a/mwmbl/background.py b/mwmbl/background.py
index c7d2b3b..d51be4a 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -7,13 +7,10 @@ from logging import getLogger, basicConfig
from pathlib import Path
from time import sleep
-from mwmbl.crawler.urls import URLDatabase
-from mwmbl.database import Database
from mwmbl.indexer import index_batches, historical
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAME
-
basicConfig(stream=sys.stdout, level=logging.INFO)
logger = getLogger(__name__)
@@ -21,9 +18,6 @@ logger = getLogger(__name__)
def run(data_path: str):
logger.info("Started background process")
- with Database() as db:
- url_db = URLDatabase(db.connection)
-
historical.run()
index_path = Path(data_path) / INDEX_NAME
batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 2edd4d8..692db79 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -15,7 +15,6 @@ from redis import Redis
from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch
from mwmbl.crawler.stats import MwmblStats, StatsManager
-from mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatus
from mwmbl.database import Database
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index b50b660..d0154f4 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -51,12 +51,18 @@ class FoundURL:
class URLDatabase:
- def __init__(self, connection):
+ def __init__(self):
+ self.urls = None
+
+ def __enter__(self):
try:
self.urls = BloomFilter.open(settings.URLS_BLOOM_FILTER_PATH)
except FileNotFoundError:
logger.info("No existing bloom filter found, creating a new one")
- self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 0.01, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)
+ self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.urls.close()
def update_found_urls(self, found_urls: list[FoundURL]):
"""
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index 10971df..329b469 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -5,18 +5,14 @@ from collections import defaultdict
from logging import getLogger
from typing import Collection, Iterable
-import spacy
-from mwmbl.indexer import process_batch
-from spacy import Language
-
from mwmbl.crawler.batch import HashedBatch, Item
-from mwmbl.crawler.urls import URLDatabase, URLStatus
-from mwmbl.database import Database
+from mwmbl.crawler.urls import URLStatus
+from mwmbl.indexer import process_batch
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.index import tokenize_document
from mwmbl.indexer.indexdb import BatchStatus
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
-from mwmbl.utils import add_term_info, add_term_infos
+from mwmbl.utils import add_term_infos
logger = getLogger(__name__)
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index f8cd6b7..6163974 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -44,8 +44,8 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qu
blacklist_retrieval_time = datetime.now() - start
logger.info(f"Recording URLs in database for {len(batches)} batches, with {len(blacklist_domains)} blacklist "
f"domains, retrieved in {blacklist_retrieval_time.total_seconds()} seconds")
- with Database() as db:
- url_db = URLDatabase(db.connection)
+
+ with URLDatabase() as url_db:
url_scores = defaultdict(float)
url_users = {}
url_timestamps = {}