An open source, non-profit web search engine
Attempt to fit a binomial distribution
Daoud Clarkeea85eb62f99f
diff --git a/analyse/estimate_unique_urls.py b/analyse/estimate_unique_urls.pynew file mode 100644index 0000000..f4f4b9c--- /dev/null+++ b/analyse/estimate_unique_urls.py@@ -0,0 +1,85 @@+"""+Estimate the number of unique URLs in the index by fitting+a Binomial distribution to the data.+"""+from collections import Counter+from pathlib import Path+from random import Random++import numpy as np+from scipy.optimize import curve_fit, differential_evolution+from scipy.stats import binom++from mwmbl.tinysearchengine.indexer import TinyIndex, Document++random = Random(1)++DEV_INDEX_PATH = Path(__file__).parent.parent / "devdata" / "index-v2.tinysearch"+++def count_unique_urls(index_path: str) -> int:+ with TinyIndex(Document, index_path) as index:+ urls = set()+ for i in range(index.num_pages):+ page = index.get_page(i)+ urls |= {doc.url for doc in page}+ if i % 1000 == 0:+ print(f"Processed {i} pages")+ return len(urls)+++def estimate_unique_urls(index_path: str, num_pages_to_sample: int = 100):+ with TinyIndex(Document, index_path) as index:+ page_sample = set()+ while len(page_sample) < num_pages_to_sample:+ page_sample.add(random.randrange(index.num_pages))++ url_counts = Counter()+ total_docs = 0+ for i in page_sample:+ page = index.get_page(i)+ url_counts.update({doc.url for doc in page})+ total_docs += len(page)++ frequencies = Counter(url_counts.values())+ print("Frequencies", frequencies)++ # Fit a binomial distribution to the data+ def binomial(x, p, s):+ print("Binomial", x, p, s)+ return binom(num_pages_to_sample, p).pmf(x) * s++ freq = np.array(list(frequencies.keys()))+ values = np.array(list(frequencies.values()))++ def cost_function(x):+ p, s = x+ value = np.sum((binomial(freq, p, s) - values) ** 2)+ return value++ result = differential_evolution(cost_function, bounds=[(0.0, 1.0), (1.0, 1e100)], args=(), maxiter=100_000, strategy='rand2bin', tol=0.001)+ print("Result", result)++ p, s = result.x+ print("Predicted values", binomial(freq, p, s))+ print("Actual", values)++ # num_unique_urls = len(url_counts)+ # mean_results_per_page = total_docs / num_pages_to_sample+ # initial_p = mean_results_per_page / num_unique_urls+ # print("Initial p", initial_p)+ # print("Initial s", num_unique_urls)+ # p0 = [initial_p, num_unique_urls]+ # f0 = binomial(freq, *p0)+ # print("F0", f0)+ #+ # p_fit, s_fit = curve_fit(binomial, freq, values, bounds=[(0.0, 1.0), (1.0, 1e100)],+ # p0=p0)[0]+ # print("Estimated parameters p, s", p_fit, s_fit)+++if __name__ == "__main__":+ estimate = estimate_unique_urls(str(DEV_INDEX_PATH), 100)++ num_urls = count_unique_urls(str(DEV_INDEX_PATH))+ print(f"Actual number of unique URLs: {num_urls}")