Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add a script to count urls in the index

Daoud Clarkeeb571fc5fee7
diff --git a/analyse/index_url_count.py b/analyse/index_url_count.py
new file mode 100644
index 0000000..f0c7ac2
--- /dev/null
+++ b/analyse/index_url_count.py
@@ -0,0 +1,19 @@
+"""
+Count unique URLs in the index.
+"""
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+
+
+def run():
+ urls = set()
+ with TinyIndex(Document, 'data/index.tinysearch') as index:
+ for i in range(index.num_pages):
+ print("Page", i)
+ page = index.get_page(i)
+ new_urls = {doc.url for doc in page}
+ urls |= new_urls
+ print("URLs", len(urls))
+
+
+if __name__ == '__main__':
+ run()