Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

1.3 KB
Raw Blame
"""
Estimate word and document frequencies from a sample of the index.
"""

import json
import os
from collections import Counter
from pathlib import Path
from urllib.parse import unquote

from mwmbl.indexer.index import prepare_url_for_tokenizing
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
from mwmbl.tokenizer import tokenize

INDEX_SAMPLE_PATH = Path(os.environ["HOME"]) / "index-sample.tinysearch"
DOCUMENT_COUNTS_PATH = Path(__file__).parent.parent / "mwmbl" / "resources" / "document_counts.json"


def get_frequencies():
    document_counts = Counter()
    with TinyIndex(item_factory=Document, index_path=INDEX_SAMPLE_PATH) as tiny_index:
        for i in range(9999):
            page = tiny_index.get_page(i)
            for document in page:
                prepared_url = prepare_url_for_tokenizing(unquote(document.url))
                for s in [document.title, prepared_url, document.extract]:
                    tokens = tokenize(s)
                    document_counts.update(set(tokens))

    print("\nDocument frequencies")
    for term, count in document_counts.most_common(200):
        print(f"{term}: {count}")

    with open(DOCUMENT_COUNTS_PATH, "w") as f:
        json.dump(document_counts, f, indent=2)


if __name__ == "__main__":
    get_frequencies()