Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Store dataset from new extension behaviour

Daoud Clarke29240d7823b4
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 959e49b..17e69eb 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -1,6 +1,7 @@
import gzip
import hashlib
import json
+from logging import getLogger
import os
from datetime import datetime, timezone, date
from queue import Empty
@@ -14,7 +15,7 @@ from ninja import NinjaAPI, Schema
from ninja.errors import HttpError
from redis import Redis
-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error
+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.database import Database
from mwmbl.indexer.batch_cache import BatchCache
@@ -38,6 +39,8 @@ from mwmbl.tinysearchengine.indexer import Document
stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))
+logger = getLogger(__name__)
+
def get_bucket(name):
s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,
@@ -46,6 +49,7 @@ def get_bucket(name):
def upload(data: bytes, name: str):
+ logger.info(f"Uploading {len(data)} bytes to {name}")
bucket = get_bucket(name)
result = bucket.put(Body=data)
return result
@@ -194,10 +198,30 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'url': f'{PUBLIC_URL_PREFIX}{filename}',
}
+ @router.post('/dataset')
+ def post_dataset(request, dataset: DatasetRequest):
+ """
+ Store a dataset of search results from Firefox extension.
+ """
+
+ if len(dataset.user_id) != USER_ID_LENGTH:
+ return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
+
+ user_id_hash = _get_user_id_hash(dataset)
+
+ now = datetime.now(timezone.utc)
+ filename = upload_object(dataset, now, user_id_hash, "dataset")
+
+ return {
+ 'status': 'ok',
+ 'public_user_id': user_id_hash,
+ 'url': f'{PUBLIC_URL_PREFIX}{filename}',
+ }
+
return router
-def _get_user_id_hash(batch: Union[Batch, NewBatchRequest]):
+def _get_user_id_hash(batch: Union[Batch, NewBatchRequest, DatasetRequest]):
return hashlib.sha3_256(batch.user_id.encode('utf8')).hexdigest()
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index 5dc8907..dd6f619 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -82,3 +82,77 @@ class PostResultsResponse(Schema):
class Error(Schema):
message: str
+
+
+class QueryDatasetEntry(Schema):
+ query: str
+ suggestion: str
+ source_term: str
+ timestamp: int
+
+
+class SearchResultEntry(Schema):
+ title: str
+ url: str
+ extract: str
+ timestamp: int
+
+
+class SearchResultSet(Schema):
+ query: str
+ results: list[SearchResultEntry]
+ timestamp: int
+ duration: int
+ success: bool
+ resultCount: int
+ searchIndex: int
+
+
+class DatasetRequest(Schema):
+ user_id: str
+ date: str
+ timestamp: int
+ extensionVersion: str
+ queryDataset: list[QueryDatasetEntry]
+ searchResults: list[SearchResultSet]
+
+
+"""
+Data format for requests to dataset backend
+
+{
+ "user_id": "user123456789012345678901234567890123456789012345678901234567890",
+ "date": "2025-01-07",
+ "timestamp": 1704672000000,
+ "extensionVersion": "0.6.1",
+ "queryDataset": [
+ {
+ "query": "wikipedia",
+ "suggestion": "wikipedia english",
+ "source_term": "wikipedia",
+ "timestamp": 1704672000000
+ }
+ // ... more entries
+ ],
+ "searchResults": [
+ {
+ "query": "wikipedia english",
+ "results": [
+ {
+ "title": "Wikipedia",
+ "url": "https://en.wikipedia.org/",
+ "extract": "Wikipedia is a free online encyclopedia...",
+ "timestamp": 1704672000000
+ }
+ // ... more results
+ ],
+ "timestamp": 1704672000000,
+ "duration": 1250,
+ "success": true,
+ "resultCount": 8,
+ "searchIndex": 1
+ }
+ // ... more searches
+ ]
+}
+"""
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 8b086aa..02171d4 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -33,7 +33,7 @@ def run():
elif mwmbl_app == "count_urls":
count_urls_continuously()
elif mwmbl_app == "server":
- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, lifespan="off", log_level="warning")
+ uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=8000, lifespan="off", log_level="warning")
else:
raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")