An open source, non-profit web search engine
Add a script to count urls in the index
Daoud Clarkeeb571fc5fee7
diff --git a/analyse/index_url_count.py b/analyse/index_url_count.pynew file mode 100644index 0000000..f0c7ac2--- /dev/null+++ b/analyse/index_url_count.py@@ -0,0 +1,19 @@+"""+Count unique URLs in the index.+"""+from mwmbl.tinysearchengine.indexer import TinyIndex, Document+++def run():+ urls = set()+ with TinyIndex(Document, 'data/index.tinysearch') as index:+ for i in range(index.num_pages):+ print("Page", i)+ page = index.get_page(i)+ new_urls = {doc.url for doc in page}+ urls |= new_urls+ print("URLs", len(urls))+++if __name__ == '__main__':+ run()