An open source, non-profit web search engine
Fix issue #60
milovanderlindendfd3f3962ec7
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 70d0614..c3e94bb 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -1,8 +1,6 @@import gzipimport hashlibimport json-import os-import refrom collections import defaultdictfrom datetime import datetime, timezone, timedelta, datefrom typing import Union@@ -18,26 +16,24 @@ from mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatusfrom mwmbl.database import Databasefrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus+from mwmbl.settings import (+ ENDPOINT_URL,+ KEY_ID,+ APPLICATION_KEY,+ BUCKET_NAME,+ MAX_BATCH_SIZE,+ USER_ID_LENGTH,+ VERSION,+ PUBLIC_URL_PREFIX,+ UNKNOWN_DOMAIN_MULTIPLIER,+ SCORE_FOR_SAME_DOMAIN,+ SCORE_FOR_DIFFERENT_DOMAIN,+ SCORE_FOR_ROOT_PATH,+ PUBLIC_USER_ID_LENGTH,+ FILE_NAME_SUFFIX,+ DATE_REGEX)from mwmbl.tinysearchengine.indexer import Document-APPLICATION_KEY = os.environ['MWMBL_APPLICATION_KEY']-KEY_ID = os.environ['MWMBL_KEY_ID']-ENDPOINT_URL = 'https://s3.us-west-004.backblazeb2.com'-BUCKET_NAME = 'mwmbl-crawl'-MAX_BATCH_SIZE = 100-USER_ID_LENGTH = 36-PUBLIC_USER_ID_LENGTH = 64-VERSION = 'v1'-DATE_REGEX = re.compile(r'\d{4}-\d{2}-\d{2}')-PUBLIC_URL_PREFIX = f'https://f004.backblazeb2.com/file/{BUCKET_NAME}/'-FILE_NAME_SUFFIX = '.json.gz'--SCORE_FOR_ROOT_PATH = 0.1-SCORE_FOR_DIFFERENT_DOMAIN = 1.0-SCORE_FOR_SAME_DOMAIN = 0.01-UNKNOWN_DOMAIN_MULTIPLIER = 0.001--router = APIRouter(prefix="/crawler", tags=["crawler"])@@ -286,4 +282,4 @@ def queue_batch(batch: HashedBatch):for item in batch.items if item.content is not None]with Database() as db:index_db = IndexDatabase(db.connection)- index_db.queue_documents(documents)\ No newline at end of file+ index_db.queue_documents(documents)diff --git a/mwmbl/database.py b/mwmbl/database.pyindex 9d8268b..bafb786 100644--- a/mwmbl/database.py+++ b/mwmbl/database.py@@ -1,14 +1,14 @@-import os-from psycopg2 import connect+from mwmbl.settings import DATABASE_URL+class Database:def __init__(self):self.connection = Nonedef __enter__(self):- self.connection = connect(os.environ["DATABASE_URL"])+ self.connection = connect(DATABASE_URL)self.connection.set_session(autocommit=True)return selfdiff --git a/mwmbl/indexer/domains.py b/mwmbl/indexer/domains.pyindex c550f2a..242df76 100644--- a/mwmbl/indexer/domains.py+++ b/mwmbl/indexer/domains.py@@ -9,14 +9,6 @@ from pathlib import Pathimport numpy as npimport pandas as pd-DATA_DIR = Path(os.environ['HOME']) / 'data' / 'tinysearch'-ALL_DOMAINS_PATH = DATA_DIR / 'hn-top-domains.csv'-TOP_DOMAINS_PATH = '../hn_top_domains_filtered.py'--MIN_COUNT = 10-PROBABILITY_THRESHOLD = 0.8--def get_top_domains():data = pd.read_csv(ALL_DOMAINS_PATH, index_col='domain')data = data[data.index.notnull()]diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.pyindex 1d703bc..58ba9e9 100644--- a/mwmbl/indexer/historical.py+++ b/mwmbl/indexer/historical.py@@ -1,7 +1,6 @@from datetime import date, timedelta-from mwmbl.crawler.app import get_user_id_hashes_for_date, get_batches_for_date_and_user, get_batch_url, \- get_batches_for_date, get_user_id_hash_from_url+from mwmbl.crawler.app import get_batches_for_date, get_user_id_hash_from_urlfrom mwmbl.database import Databasefrom mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabasediff --git a/mwmbl/settings.py b/mwmbl/settings.pynew file mode 100644index 0000000..83b8032--- /dev/null+++ b/mwmbl/settings.py@@ -0,0 +1,29 @@+import os+import re+from pathlib import Path+++DATA_DIR = Path(os.environ['HOME']) / 'data' / 'tinysearch'+ALL_DOMAINS_PATH = DATA_DIR / 'hn-top-domains.csv'+TOP_DOMAINS_PATH = '../hn_top_domains_filtered.py'++MIN_COUNT = 10+PROBABILITY_THRESHOLD = 0.8+DATABASE_URL = os.environ.get("DATABASE_URL", "postgres://username:password@localhost/dbname")++APPLICATION_KEY = os.environ.get('MWMBL_APPLICATION_KEY', 'SECRETAPPLICATIONKEY')+KEY_ID = os.environ.get('MWMBL_KEY_ID', 'SECRETKEYID')+ENDPOINT_URL = os.environ.get('MWMBL_ENDPOINT_URL', 'https://s3.us-west-004.backblazeb2.com')+BUCKET_NAME = os.environ.get('MWMBL_BUCKET_NAME', 'mwmbl-crawl')+MAX_BATCH_SIZE = 100+USER_ID_LENGTH = 36+PUBLIC_USER_ID_LENGTH = 64+VERSION = 'v1'+DATE_REGEX = re.compile(r'\d{4}-\d{2}-\d{2}')+PUBLIC_URL_PREFIX = f'https://f004.backblazeb2.com/file/{BUCKET_NAME}/'+FILE_NAME_SUFFIX = '.json.gz'++SCORE_FOR_ROOT_PATH = 0.1+SCORE_FOR_DIFFERENT_DOMAIN = 1.0+SCORE_FOR_SAME_DOMAIN = 0.01+UNKNOWN_DOMAIN_MULTIPLIER = 0.001