An open source, non-profit web search engine
Use new server
Daoud Clarke14107acc754b
diff --git a/analyse/record_historical_batches.py b/analyse/record_historical_batches.pyindex 6c8c80e..4d8ccd3 100644--- a/analyse/record_historical_batches.py+++ b/analyse/record_historical_batches.py@@ -12,21 +12,44 @@ import requestsfrom mwmbl.indexer.paths import CRAWL_GLOB-API_ENDPOINT = "http://localhost:8080/batches/historical"+API_ENDPOINT = "http://95.216.215.29/batches/historical"+++def total_num_batches():+ return len(glob.glob(CRAWL_GLOB))def get_batches():- for path in glob.glob(CRAWL_GLOB):+ for path in sorted(glob.glob(CRAWL_GLOB)):hashed_batch = json.load(gzip.open(path))yield hashed_batch+def convert_item(item):+ return {+ 'url': item['url'],+ 'status': 200,+ 'timestamp': item['timestamp'],+ 'content': {+ 'title': item['title'],+ 'extract': item['extract'],+ 'links': item['links'],+ }+ }+++def run():+ total_batches = total_num_batches()batches = get_batches()- for hashed_batch in batches:- print("Recording batch", hashed_batch)- response = requests.post(API_ENDPOINT, json=hashed_batch)- print("Response", response)+ for i, hashed_batch in enumerate(batches):+ new_batch = {+ 'user_id_hash': hashed_batch['user_id_hash'],+ 'timestamp': hashed_batch['timestamp'],+ 'items': [convert_item(item) for item in hashed_batch['items']]+ }+ response = requests.post(API_ENDPOINT, json=new_batch)+ print(f"Response {i} of {total_batches}", response)if __name__ == '__main__':