An open source, non-profit web search engine
Delete unused SQL
Daoud Clarke218d87365442
diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.pyindex 8398f0a..9386680 100644--- a/mwmbl/indexer/historical.py+++ b/mwmbl/indexer/historical.py@@ -4,7 +4,7 @@ from mwmbl.crawler.app import get_batches_for_datefrom mwmbl.database import Databasefrom mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase-DAYS = 10+DAYS = 20def run():diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 8256bef..eabc567 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -36,7 +36,7 @@ def run(batch_cache: BatchCache, index_path: str):index_db = IndexDatabase(db.connection)logger.info("Getting local batches")- batches = index_db.get_batches_by_status(BatchStatus.LOCAL, 1000)+ batches = index_db.get_batches_by_status(BatchStatus.LOCAL, 10000)logger.info(f"Got {len(batches)} batch urls")if len(batches) == 0:returndiff --git a/mwmbl/indexer/indexdb.py b/mwmbl/indexer/indexdb.pyindex 2c02317..8bff0ec 100644--- a/mwmbl/indexer/indexdb.py+++ b/mwmbl/indexer/indexdb.py@@ -13,11 +13,6 @@ class BatchStatus(Enum):INDEXED = 2-class DocumentStatus(Enum):- NEW = 0- PREPROCESSING = 1--@dataclassclass BatchInfo:url: str@@ -38,32 +33,8 @@ class IndexDatabase:)"""- documents_sql = """- CREATE TABLE IF NOT EXISTS documents (- url VARCHAR PRIMARY KEY,- title VARCHAR NOT NULL,- extract VARCHAR NOT NULL,- score FLOAT NOT NULL,- status INT NOT NULL- )- """-- document_pages_sql = """- CREATE TABLE IF NOT EXISTS document_pages (- url VARCHAR NOT NULL,- page INT NOT NULL- )- """-- document_pages_index_sql = """- CREATE INDEX IF NOT EXISTS document_pages_page_index ON document_pages (page)- """-with self.connection.cursor() as cursor:cursor.execute(batches_sql)- cursor.execute(documents_sql)- cursor.execute(document_pages_sql)- cursor.execute(document_pages_index_sql)def record_batches(self, batch_infos: list[BatchInfo]):sql = """