An open source, non-profit web search engine
Investigate duplication of URLs in batches
Daoud Clarkee27d749e18c5
diff --git a/analyse/analyse_crawled_domains.py b/analyse/analyse_crawled_domains.pyindex a3d36b1..371cbb6 100644--- a/analyse/analyse_crawled_domains.py+++ b/analyse/analyse_crawled_domains.py@@ -7,15 +7,23 @@ import jsonfrom collections import defaultdict, Counterfrom urllib.parse import urlparse-from mwmbl.indexer.paths import CRAWL_GLOB+from mwmbl.crawler.batch import HashedBatch+from mwmbl.indexer.paths import CRAWL_GLOB, MWMBL_DATA_DIR+++# TODO: remove this line - temporary override+CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/2022-06-23/*/*/*.json.gz"def get_urls():for path in glob.glob(CRAWL_GLOB):data = json.load(gzip.open(path))- user = data['user_id_hash']- for item in data['items']:- yield user, item['url']+ batch = HashedBatch.parse_obj(data)+ user = batch.user_id_hash+ for item in batch.items:+ if item.content is not None:+ for url in item.content.links:+ yield user, urldef analyse_urls(urls):diff --git a/analyse/recent_batches.py b/analyse/recent_batches.pynew file mode 100644index 0000000..460caba--- /dev/null+++ b/analyse/recent_batches.py@@ -0,0 +1,4 @@+"""+Analyse recent batches looking for duplicates.+"""+diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 1108339..431b2fa 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -4,14 +4,14 @@ import jsonimport osimport refrom datetime import datetime, timezone, timedelta-from typing import Optional, Union+from typing import Unionfrom uuid import uuid4import boto3import requestsfrom fastapi import HTTPException, APIRouter-from pydantic import BaseModel+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatchfrom mwmbl.crawler.urls import URLDatabasefrom mwmbl.database import Database@@ -43,40 +43,6 @@ def upload(data: bytes, name: str):return result-class ItemContent(BaseModel):- title: str- extract: str- links: list[str]---class ItemError(BaseModel):- name: str- message: Optional[str]---class Item(BaseModel):- url: str- status: Optional[int]- timestamp: int- content: Optional[ItemContent]- error: Optional[ItemError]---class Batch(BaseModel):- user_id: str- items: list[Item]---class NewBatchRequest(BaseModel):- user_id: str---class HashedBatch(BaseModel):- user_id_hash: str- timestamp: int- items: list[Item]--last_batch = None@@ -232,7 +198,6 @@ def get_subfolders(prefix):items = client.list_objects(Bucket=BUCKET_NAME,Prefix=prefix,Delimiter='/')- print("Got items", items)item_keys = [item['Prefix'][len(prefix):].strip('/') for item in items['CommonPrefixes']]return item_keysdiff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pynew file mode 100644index 0000000..1e95a6a--- /dev/null+++ b/mwmbl/crawler/batch.py@@ -0,0 +1,37 @@+from typing import Optional++from pydantic import BaseModel+++class ItemContent(BaseModel):+ title: str+ extract: str+ links: list[str]+++class ItemError(BaseModel):+ name: str+ message: Optional[str]+++class Item(BaseModel):+ url: str+ status: Optional[int]+ timestamp: int+ content: Optional[ItemContent]+ error: Optional[ItemError]+++class Batch(BaseModel):+ user_id: str+ items: list[Item]+++class NewBatchRequest(BaseModel):+ user_id: str+++class HashedBatch(BaseModel):+ user_id_hash: str+ timestamp: int+ items: list[Item]diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.pyindex 42e679d..447c92a 100644--- a/mwmbl/indexer/historical.py+++ b/mwmbl/indexer/historical.py@@ -3,7 +3,8 @@ from datetime import date, datetime, timedeltaimport spacyfrom mwmbl.crawler.app import get_user_id_hashes_for_date, get_batches_for_date_and_user, get_batch_from_id, \- create_historical_batch, HashedBatch, get_batch_url+ create_historical_batch, get_batch_url+from mwmbl.crawler.batch import HashedBatchfrom mwmbl.database import Databasefrom mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabasefrom mwmbl.indexer.index import tokenize_document@@ -17,13 +18,13 @@ def run():for day in range(DAYS):date_str = str(date.today() - timedelta(days=day))users = get_user_id_hashes_for_date(date_str)- print("Users", users)+ print(f"Got {len(users)} for day {date_str}")with Database() as db:index_db = IndexDatabase(db.connection)index_db.create_tables()for user in users:batches = get_batches_for_date_and_user(date_str, user)- print("Batches", batches)+ print("Historical batches for user", user, len(batches))batch_urls = [get_batch_url(batch_id, date_str, user) for batch_id in batches["batch_ids"]]infos = [BatchInfo(url, user, BatchStatus.REMOTE) for url in batch_urls]index_db.record_batches(infos)diff --git a/mwmbl/indexer/indexdb.py b/mwmbl/indexer/indexdb.pyindex f8604e9..9c7d663 100644--- a/mwmbl/indexer/indexdb.py+++ b/mwmbl/indexer/indexdb.py@@ -12,7 +12,6 @@ from mwmbl.tinysearchengine.indexer import Documentclass BatchStatus(Enum):REMOTE = 0 # The batch only exists in long term storageLOCAL = 1 # We have a copy of the batch locally in Postgresql- INDEXED = 2 # The batch has been indexed and the local data has been deletedclass DocumentStatus(Enum):@@ -108,6 +107,7 @@ class IndexDatabase:data = [(document.url, document.title, document.extract, document.score, DocumentStatus.NEW.value)for document in sorted_documents]+ print("Queueing documents", len(data))with self.connection.cursor() as cursor:execute_values(cursor, sql, data)@@ -126,7 +126,6 @@ class IndexDatabase:with self.connection.cursor() as cursor:cursor.execute(sql)results = cursor.fetchall()- print("Results", results)return [Document(title, url, extract, score) for url, title, extract, score in results]def queue_documents_for_page(self, urls_and_page_indexes: list[tuple[str, int]]):@@ -134,7 +133,7 @@ class IndexDatabase:INSERT INTO document_pages (url, page) values %s"""- print("Queuing", urls_and_page_indexes)+ print(f"Queuing {len(urls_and_page_indexes)} urls and page indexes")with self.connection.cursor() as cursor:execute_values(cursor, sql, urls_and_page_indexes)diff --git a/mwmbl/indexer/preprocess.py b/mwmbl/indexer/preprocess.pyindex 53ad2fa..858d517 100644--- a/mwmbl/indexer/preprocess.py+++ b/mwmbl/indexer/preprocess.py@@ -27,7 +27,7 @@ def run_preprocessing(index_path):with Database() as db:index_db = IndexDatabase(db.connection)documents = index_db.get_documents_for_preprocessing()- print(f"Got {len(documents)} documents")+ print(f"Got {len(documents)} documents for preprocessing")if len(documents) == 0:sleep(10)with TinyIndex(Document, index_path, 'w') as indexer:diff --git a/mwmbl/indexer/retrieve.py b/mwmbl/indexer/retrieve.pyindex 1e8f9ec..b03efd0 100644--- a/mwmbl/indexer/retrieve.py+++ b/mwmbl/indexer/retrieve.py@@ -9,7 +9,7 @@ from time import sleepimport requests-from mwmbl.crawler.app import HashedBatch+from mwmbl.crawler.batch import HashedBatchfrom mwmbl.database import Databasefrom mwmbl.indexer.indexdb import IndexDatabase, BatchStatusfrom mwmbl.retry import retry_requests@@ -26,21 +26,19 @@ def retrieve_batches():with Database() as db:index_db = IndexDatabase(db.connection)batches = index_db.get_batches_by_status(BatchStatus.REMOTE)- print("Batches", batches)+ print(f"Found {len(batches)} remote batches")urls = [batch.url for batch in batches]pool = ThreadPool(NUM_THREADS)results = pool.imap_unordered(retrieve_batch, urls)for result in results:print("Processed batch with items:", result)-- with Database() as db:- index_db = IndexDatabase(db.connection)- index_db.update_batch_status(urls, BatchStatus.LOCAL)+ index_db.update_batch_status(urls, BatchStatus.LOCAL)def retrieve_batch(url):data = json.loads(gzip.decompress(retry_requests.get(url).content))batch = HashedBatch.parse_obj(data)+ print(f"Retrieved batch with {len(batch.items)} items")queue_batch(batch)return len(batch.items)diff --git a/mwmbl/indexer/update_pages.py b/mwmbl/indexer/update_pages.pyindex 051eb0e..d3fce85 100644--- a/mwmbl/indexer/update_pages.py+++ b/mwmbl/indexer/update_pages.py@@ -18,21 +18,20 @@ def run_update(index_path):for i in range(indexer.num_pages):with Database() as db:index_db = IndexDatabase(db.connection)- pages = index_db.get_queued_documents_for_page(i)- if len(pages) > 0:- print("Pages", len(pages))- else:+ documents = index_db.get_queued_documents_for_page(i)+ print(f"Documents queued for page {i}: {len(documents)}")+ if len(documents) == 0:continuefor j in range(3):try:- indexer.add_to_page(i, pages)+ indexer.add_to_page(i, documents)breakexcept ValueError:- pages = pages[:len(pages)//2]- if len(pages) == 0:+ documents = documents[:len(documents)//2]+ if len(documents) == 0:break- print(f"Not enough space, adding {len(pages)}")+ print(f"Not enough space, adding {len(documents)}")index_db.clear_queued_documents_for_page(i)diff --git a/mwmbl/main.py b/mwmbl/main.pyindex fd8a7d9..5f78c09 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -6,7 +6,7 @@ from multiprocessing import Processimport uvicornfrom fastapi import FastAPI-from mwmbl.indexer import historical, retrieve, preprocess+from mwmbl.indexer import historical, retrieve, preprocess, update_pagesfrom mwmbl.crawler.app import router as crawler_routerfrom mwmbl.tinysearchengine import searchfrom mwmbl.tinysearchengine.completer import Completer@@ -43,6 +43,7 @@ def run():Process(target=historical.run).start()Process(target=retrieve.run).start()Process(target=preprocess.run, args=(args.index,)).start()+ Process(target=update_pages.run, args=(args.index,)).start()completer = Completer()