An open source, non-profit web search engine
import os
import pickle
from datetime import datetime, timezone
from pathlib import Path
from queue import Queue
from mwmbl.indexer import record_urls_in_database
def run_update_urls_on_fixed_batches():
with open(Path(os.environ["HOME"]) / "data" / "mwmbl" / "hashed-batches.pickle", "rb") as file:
batches = pickle.load(file)
# print("Batches", batches[:3])
queue = Queue()
start = datetime.now(timezone.utc)
record_urls_in_database(batches, queue)
total_time = (datetime.now(timezone.utc) - start).total_seconds()
print("Total time:", total_time)
if __name__ == "__main__":
run_update_urls_on_fixed_batches()