Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

1.1 KB
Raw Blame
"""
Export the list of unique URLs to a SQLite file for analysis/evaluation.
"""

import sqlite3

from mwmbl.app import get_config_and_index
from mwmbl.indexer import URLS_PATH


def create_database():
    with sqlite3.connect(URLS_PATH) as connection:
        connection.execute("""
            CREATE TABLE urls (url TEXT PRIMARY KEY)
        """)


def get_url_batches():
    config, index = get_config_and_index()
    for page_num in range(config.index_config.num_pages):
        if page_num % 1000 == 0:
            print("Processing page", page_num)
        page = index.get_page(page_num)
        if page is None:
            continue
        yield [url for title, url, extract in page]


def run():
    create_database()
    url_batches = get_url_batches()

    with sqlite3.connect(URLS_PATH) as connection:
        for url_batch in url_batches:
            parameters = [(url,) for url in url_batch]
            connection.executemany(
                """
            INSERT OR IGNORE INTO urls VALUES (?)
            """,
                parameters,
            )


if __name__ == "__main__":
    run()