Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

660 B
Raw Blame
import os
import pickle
from datetime import datetime, timezone
from pathlib import Path
from queue import Queue

from mwmbl.indexer import record_urls_in_database


def run_update_urls_on_fixed_batches():
    with open(Path(os.environ["HOME"]) / "data" / "mwmbl" / "hashed-batches.pickle", "rb") as file:
        batches = pickle.load(file)

    # print("Batches", batches[:3])

    queue = Queue()

    start = datetime.now(timezone.utc)
    record_urls_in_database(batches, queue)
    total_time = (datetime.now(timezone.utc) - start).total_seconds()

    print("Total time:", total_time)


if __name__ == "__main__":
    run_update_urls_on_fixed_batches()