Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge branch 'main' into new-crawler

Daoud Clarkee2aec51eb471
diff --cc devdata/index-v2.tinysearch
index 4a1f152,490c48e..1508411
Binary files differ
diff --cc mwmbl/crawler/app.py
index cd261e6,b0e6ba5..c6540f7
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@@ -111,29 -118,16 +111,31 @@@ def create_router(batch_cache: BatchCac
'url': batch_url,
}
+ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):
+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
+
+ # How to pad a string with zeros: https://stackoverflow.com/a/39402910
+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough
+ padded_seconds = str(seconds).zfill(5)
+
+ # See discussion here: https://stackoverflow.com/a/13484764
+ uid = str(uuid4())[:8]
+
+ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'
+ data = gzip.compress(model_object.json().encode('utf8'))
+ upload(data, filename)
+ return filename
+
@router.post('/batches/new')
def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:
- user_id_hash = _get_user_id_hash(batch_request)
- try:
- urls = queued_batches.get_batch(user_id_hash)
- except Empty:
- return []
- return urls
- # TODO: temporarily disable crawling
- return []
+ # user_id_hash = _get_user_id_hash(batch_request)
+ # try:
+ # urls = queued_batches.get_batch(user_id_hash)
+ # except Empty:
+ # return []
+ # return urls
++ # TODO: temporarily disable crawling
++ return []
@router.get('/batches/{date_str}/users/{public_user_id}')
def get_batches_for_date_and_user(request, date_str, public_user_id):