An open source, non-profit web search engine
Store dataset from new extension behaviour
Daoud Clarke29240d7823b4
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 959e49b..17e69eb 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -1,6 +1,7 @@import gzipimport hashlibimport json+from logging import getLoggerimport osfrom datetime import datetime, timezone, datefrom queue import Empty@@ -14,7 +15,7 @@ from ninja import NinjaAPI, Schemafrom ninja.errors import HttpErrorfrom redis import Redis-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequestfrom mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Databasefrom mwmbl.indexer.batch_cache import BatchCache@@ -38,6 +39,8 @@ from mwmbl.tinysearchengine.indexer import Documentstats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))+logger = getLogger(__name__)+def get_bucket(name):s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,@@ -46,6 +49,7 @@ def get_bucket(name):def upload(data: bytes, name: str):+ logger.info(f"Uploading {len(data)} bytes to {name}")bucket = get_bucket(name)result = bucket.put(Body=data)return result@@ -194,10 +198,30 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'url': f'{PUBLIC_URL_PREFIX}{filename}',}+ @router.post('/dataset')+ def post_dataset(request, dataset: DatasetRequest):+ """+ Store a dataset of search results from Firefox extension.+ """++ if len(dataset.user_id) != USER_ID_LENGTH:+ return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)++ user_id_hash = _get_user_id_hash(dataset)++ now = datetime.now(timezone.utc)+ filename = upload_object(dataset, now, user_id_hash, "dataset")++ return {+ 'status': 'ok',+ 'public_user_id': user_id_hash,+ 'url': f'{PUBLIC_URL_PREFIX}{filename}',+ }+return router-def _get_user_id_hash(batch: Union[Batch, NewBatchRequest]):+def _get_user_id_hash(batch: Union[Batch, NewBatchRequest, DatasetRequest]):return hashlib.sha3_256(batch.user_id.encode('utf8')).hexdigest()diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex 5dc8907..dd6f619 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -82,3 +82,77 @@ class PostResultsResponse(Schema):class Error(Schema):message: str+++class QueryDatasetEntry(Schema):+ query: str+ suggestion: str+ source_term: str+ timestamp: int+++class SearchResultEntry(Schema):+ title: str+ url: str+ extract: str+ timestamp: int+++class SearchResultSet(Schema):+ query: str+ results: list[SearchResultEntry]+ timestamp: int+ duration: int+ success: bool+ resultCount: int+ searchIndex: int+++class DatasetRequest(Schema):+ user_id: str+ date: str+ timestamp: int+ extensionVersion: str+ queryDataset: list[QueryDatasetEntry]+ searchResults: list[SearchResultSet]+++"""+Data format for requests to dataset backend++{+ "user_id": "user123456789012345678901234567890123456789012345678901234567890",+ "date": "2025-01-07",+ "timestamp": 1704672000000,+ "extensionVersion": "0.6.1",+ "queryDataset": [+ {+ "query": "wikipedia",+ "suggestion": "wikipedia english",+ "source_term": "wikipedia",+ "timestamp": 1704672000000+ }+ // ... more entries+ ],+ "searchResults": [+ {+ "query": "wikipedia english",+ "results": [+ {+ "title": "Wikipedia",+ "url": "https://en.wikipedia.org/",+ "extract": "Wikipedia is a free online encyclopedia...",+ "timestamp": 1704672000000+ }+ // ... more results+ ],+ "timestamp": 1704672000000,+ "duration": 1250,+ "success": true,+ "resultCount": 8,+ "searchIndex": 1+ }+ // ... more searches+ ]+}+"""diff --git a/mwmbl/main.py b/mwmbl/main.pyindex 8b086aa..02171d4 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -33,7 +33,7 @@ def run():elif mwmbl_app == "count_urls":count_urls_continuously()elif mwmbl_app == "server":- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, lifespan="off", log_level="warning")+ uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=8000, lifespan="off", log_level="warning")else:raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")