An open source, non-profit web search engine
Merge branch 'main' into new-crawler
Daoud Clarkee2aec51eb471
diff --cc devdata/index-v2.tinysearchindex 4a1f152,490c48e..1508411Binary files differdiff --cc mwmbl/crawler/app.pyindex cd261e6,b0e6ba5..c6540f7--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@@ -111,29 -118,16 +111,31 @@@ def create_router(batch_cache: BatchCac'url': batch_url,}+ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds++ # How to pad a string with zeros: https://stackoverflow.com/a/39402910+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough+ padded_seconds = str(seconds).zfill(5)++ # See discussion here: https://stackoverflow.com/a/13484764+ uid = str(uuid4())[:8]++ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'+ data = gzip.compress(model_object.json().encode('utf8'))+ upload(data, filename)+ return filename+@router.post('/batches/new')def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:- user_id_hash = _get_user_id_hash(batch_request)- try:- urls = queued_batches.get_batch(user_id_hash)- except Empty:- return []- return urls- # TODO: temporarily disable crawling- return []+ # user_id_hash = _get_user_id_hash(batch_request)+ # try:+ # urls = queued_batches.get_batch(user_id_hash)+ # except Empty:+ # return []+ # return urls++ # TODO: temporarily disable crawling++ return []@router.get('/batches/{date_str}/users/{public_user_id}')def get_batches_for_date_and_user(request, date_str, public_user_id):