An open source, non-profit web search engine
Store hashed user ID instead of raw one
Daoud Clarkef1232837f93b
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 17e69eb..991aacb 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -15,7 +15,7 @@ from ninja import NinjaAPI, Schemafrom ninja.errors import HttpErrorfrom redis import Redis-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest, HashedDatasetfrom mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Databasefrom mwmbl.indexer.batch_cache import BatchCache@@ -209,8 +209,18 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiouser_id_hash = _get_user_id_hash(dataset)+ # Create a hashed dataset that doesn't contain the raw user_id+ hashed_dataset = HashedDataset(+ user_id_hash=user_id_hash,+ date=dataset.date,+ timestamp=dataset.timestamp,+ extensionVersion=dataset.extensionVersion,+ queryDataset=dataset.queryDataset,+ searchResults=dataset.searchResults+ )+now = datetime.now(timezone.utc)- filename = upload_object(dataset, now, user_id_hash, "dataset")+ filename = upload_object(hashed_dataset, now, user_id_hash, "dataset")return {'status': 'ok',diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex dd6f619..f5d266f 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -117,6 +117,15 @@ class DatasetRequest(Schema):searchResults: list[SearchResultSet]+class HashedDataset(Schema):+ user_id_hash: str+ date: str+ timestamp: int+ extensionVersion: str+ queryDataset: list[QueryDatasetEntry]+ searchResults: list[SearchResultSet]++"""Data format for requests to dataset backend