An open source, non-profit web search engine
Estimate counts instead of using a hyperloglog
Daoud Clarked2448ae10130
diff --git a/analyse/estimate_unique_urls.py b/analyse/estimate_unique_urls.pyindex ec6d821..dd38c5f 100644--- a/analyse/estimate_unique_urls.py+++ b/analyse/estimate_unique_urls.py@@ -7,8 +7,11 @@ from pathlib import Pathfrom random import Randomimport numpy as np+from pydistinct.ensemble_estimators import median_estimator+from pydistinct.stats_estimators import bootstrap_estimator, goodmans_estimator, smoothed_jackknife_estimator, \+ horvitz_thompson_estimatorfrom scipy.optimize import curve_fit, differential_evolution-from scipy.stats import binom, poisson+from scipy.stats import binom, poisson, betabinomfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -41,58 +44,11 @@ def estimate_unique_urls(index_path: str, num_pages_to_sample: int = 100):url_counts.update({doc.url for doc in page})total_docs += len(page)- frequencies = Counter(url_counts.values())- print("Frequencies", frequencies)-- # Fit a binomial distribution to the data- def binomial(x, p, s):- print("Binomial", x, p, s)- return binom(num_pages_to_sample, p).pmf(x) * s-- frequencies = dict(sorted(frequencies.items()))- freq = np.array(list(frequencies.keys()))- values = np.array(list(frequencies.values()))-- def poiss(x, m, s):- return poisson.pmf(x, m) * s-- m_fit, s_fit = curve_fit(poiss, freq, values)[0]- print("Estimated parameter m", m_fit, s_fit)-- predictions = poiss(freq, m_fit, s_fit)- print("Predictions", predictions)- print("Actual", values)--- #- # def cost_function(x):- # p, s = x- # value = np.sum((binomial(freq, p, s) - values) ** 2)- # return value- #- # result = differential_evolution(cost_function, bounds=[(0.0, 1.0), (1.0, 1e100)], args=(), maxiter=100_000, strategy='rand2bin', tol=0.001)- # print("Result", result)- #- # p, s = result.x- # print("Predicted values", binomial(freq, p, s))- # print("Actual", values)-- # num_unique_urls = len(url_counts)- # mean_results_per_page = total_docs / num_pages_to_sample- # initial_p = mean_results_per_page / num_unique_urls- # print("Initial p", initial_p)- # print("Initial s", num_unique_urls)- # p0 = [initial_p, num_unique_urls]- # f0 = binomial(freq, *p0)- # print("F0", f0)- #- # p_fit, s_fit = curve_fit(binomial, freq, values, bounds=[(0.0, 1.0), (1.0, 1e100)],- # p0=p0)[0]- # print("Estimated parameters p, s", p_fit, s_fit)+ return median_estimator(attributes=dict(url_counts.items()))if __name__ == "__main__":- estimate = estimate_unique_urls(str(DEV_INDEX_PATH), 1000)+ estimate = estimate_unique_urls(str(DEV_INDEX_PATH), 500)num_urls = count_unique_urls(str(DEV_INDEX_PATH))print(f"Actual number of unique URLs: {num_urls}")diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.pyindex 57d8afa..fd0fb02 100644--- a/mwmbl/count_urls.py+++ b/mwmbl/count_urls.py@@ -2,29 +2,29 @@ import osfrom collections import Counterfrom datetime import date, timedelta, datetimefrom logging import getLogger-from multiprocessing import Poolfrom pathlib import Path+from random import Randomfrom time import sleepfrom django.conf import settings+from pydistinct.ensemble_estimators import median_estimatorfrom redis import Redisfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document-from mwmbl.utils import batch, parse_url+from mwmbl.utils import parse_urlINDEX_RESULT_COUNT_KEY = "index-result-count-{date}"INDEX_DOMAIN_COUNT_KEY = "index-domain-count-{date}"INDEX_URL_COUNT_KEY = "index-url-count-{date}"INDEX_DOMAIN_RESULT_COUNT_KEY = "index-domain-result-count-{date}"-INDEX_URL_HLL_KEY = "index-hll-{date}"-INDEX_DOMAIN_HLL_KEY = "index-domain-hll-{date}"-SHORT_EXPIRE_SECONDS = 60 * 60 * 24LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30-NUM_PAGES_IN_BATCH = 10240++PAGE_PROPORTION_TO_SAMPLE = 0.0001logger = getLogger(__name__)+random = Random(1)def get_redis():@@ -43,75 +43,44 @@ def count_urls_continuously():def count_urls():- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:- page_indexes = batch(range(tiny_index.num_pages), NUM_PAGES_IN_BATCH)-start_time = datetime.utcnow()- with Pool(processes=4) as pool:- num_results = sum(pool.map(count_urls_in_batch, page_indexes))- logger.info(f"Processed {num_results} results in total.")-- domain_count_key, domain_hll_key, url_hll_key = _get_keys()+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME+ with TinyIndex(item_factory=Document, index_path=index_path) as index:+ page_sample = set()+ num_pages_to_sample = max(100, int(index.num_pages * PAGE_PROPORTION_TO_SAMPLE))+ logger.info(f"Sampling {num_pages_to_sample} pages.")+ while len(page_sample) < num_pages_to_sample:+ page_sample.add(random.randrange(index.num_pages))++ url_counts = Counter()+ domain_counts = Counter()+ total_docs = 0+ for i in page_sample:+ page = index.get_page(i)+ url_counts.update({doc.url for doc in page})+ domains = [parse_url(doc.url).netloc for doc in page]+ domain_counts.update(domains)+ total_docs += len(page)++ url_count_estimate = median_estimator(attributes=dict(url_counts.items()))+ domain_count_estimate = median_estimator(attributes=dict(domain_counts.items()))+ num_results_estimate = total_docs / PAGE_PROPORTION_TO_SAMPLE++ logger.info(f"Estimated {url_count_estimate} unique URLs, {domain_count_estimate} unique domains, "+ f"and {num_results_estimate} results in the index.")redis = get_redis()- redis.expire(url_hll_key, SHORT_EXPIRE_SECONDS)- url_count = redis.pfcount(url_hll_key)- logger.info("Counted %d unique URLs", url_count)-- redis.expire(domain_hll_key, SHORT_EXPIRE_SECONDS)- domain_count = redis.pfcount(domain_hll_key)- logger.info("Counted %d unique domains", domain_count)-- redis.expire(domain_count_key, SHORT_EXPIRE_SECONDS)today = date.today()- _set_count(INDEX_URL_COUNT_KEY, redis, today, url_count)- _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, domain_count)- _set_count(INDEX_RESULT_COUNT_KEY, redis, today, num_results)+ _set_count(INDEX_URL_COUNT_KEY, redis, today, int(url_count_estimate))+ _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, int(domain_count_estimate))+ _set_count(INDEX_RESULT_COUNT_KEY, redis, today, int(num_results_estimate))end_time = datetime.utcnow()logger.info(f"Counting took {end_time - start_time}.")-def count_urls_in_batch(page_indexes: list[int]):- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME- redis = get_redis()-- logger.info(f"Counting URLs in index {index_path}")-- domain_count_key, domain_hll_key, url_hll_key = _get_keys()- num_results = 0- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:- urls = set()- domains = set()- domain_counts = Counter()- for i in page_indexes:- docs = tiny_index.get_page(i)- urls |= {doc.url for doc in docs}- page_domains = [parse_url(doc.url).netloc for doc in docs]- domain_counts.update(page_domains)- domains |= set(page_domains)- num_results += len(docs)-- for domain, count in domain_counts.items():- redis.zincrby(domain_count_key, count, domain)-- redis.pfadd(url_hll_key, *urls)- redis.pfadd(domain_hll_key, *domains)- logger.info(f"Counted {i} pages of {tiny_index.num_pages}.")- return num_results---def _get_keys():- today = date.today()- url_hll_key = INDEX_URL_HLL_KEY.format(date=today)- domain_hll_key = INDEX_DOMAIN_HLL_KEY.format(date=today)- domain_count_key = INDEX_DOMAIN_RESULT_COUNT_KEY.format(date=today)- return domain_count_key, domain_hll_key, url_hll_key--def _set_count(key, redis, today, count):redis.set(key.format(date=today), count)redis.expire(key.format(date=today), LONG_EXPIRE_SECONDS)diff --git a/poetry.lock b/poetry.lockindex 2e44154..68bbe78 100644--- a/poetry.lock+++ b/poetry.lock@@ -620,6 +620,17 @@ Django = ">=1.11"[package.extras]dev = ["black", "flake8"]+[[package]]+name = "docutils"+version = "0.21.2"+description = "Docutils -- Python Documentation Utilities"+optional = false+python-versions = ">=3.9"+files = [+ {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"},+ {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"},+]+[[package]]name = "exceptiongroup"version = "1.2.0"@@ -1094,6 +1105,20 @@ files = [[package.dependencies]lxml = "*"+[[package]]+name = "m2r"+version = "0.3.1"+description = "Markdown and reStructuredText in a single file."+optional = false+python-versions = "*"+files = [+ {file = "m2r-0.3.1.tar.gz", hash = "sha256:aafb67fc49cfb1d89e46a3443ac747e15f4bb42df20ed04f067ad9efbee256ab"},+]++[package.dependencies]+docutils = "*"+mistune = "<2"+[[package]]name = "markupsafe"version = "2.1.5"@@ -1163,6 +1188,17 @@ files = [{file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"},]+[[package]]+name = "mistune"+version = "0.8.4"+description = "The fastest markdown parser in pure Python"+optional = false+python-versions = "*"+files = [+ {file = "mistune-0.8.4-py2.py3-none-any.whl", hash = "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"},+ {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"},+]+[[package]]name = "mmh3"version = "3.1.0"@@ -1286,6 +1322,16 @@ files = [{file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"},]+[[package]]+name = "nvidia-nccl-cu12"+version = "2.22.3"+description = "NVIDIA Collective Communication Library (NCCL) Runtime"+optional = false+python-versions = ">=3"+files = [+ {file = "nvidia_nccl_cu12-2.22.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9f5e03c00269dee2cd1aa57019f9a024478a74ae6e9b32d5341c849fe6f6302"},+]+[[package]]name = "oauthlib"version = "3.2.2"@@ -1405,6 +1451,24 @@ gcs = ["google-cloud-storage (>=1.26.0,<2.0.0)"]s3 = ["boto3"]test = ["mock", "pytest", "pytest-coverage", "typer-cli"]+[[package]]+name = "patsy"+version = "0.5.6"+description = "A Python package for describing statistical models and for building design matrices."+optional = false+python-versions = "*"+files = [+ {file = "patsy-0.5.6-py2.py3-none-any.whl", hash = "sha256:19056886fd8fa71863fa32f0eb090267f21fb74be00f19f5c70b2e9d76c883c6"},+ {file = "patsy-0.5.6.tar.gz", hash = "sha256:95c6d47a7222535f84bff7f63d7303f2e297747a598db89cf5c67f0c0c7d2cdb"},+]++[package.dependencies]+numpy = ">=1.4"+six = "*"++[package.extras]+test = ["pytest", "pytest-cov", "scipy"]+[[package]]name = "platformdirs"version = "4.2.0"@@ -1709,6 +1773,24 @@ typing-extensions = ">=3.7.4.3"dotenv = ["python-dotenv (>=0.10.4)"]email = ["email-validator (>=1.0.3)"]+[[package]]+name = "pydistinct"+version = "0.6.4"+description = "Package for estimating distinct values in a population"+optional = false+python-versions = "*"+files = [+ {file = "pydistinct-0.6.4-py3-none-any.whl", hash = "sha256:c5089aa0cad9b8bbf26e2735cd5cdd69289a8d3ce82e9b740335feab5a3ce149"},+ {file = "pydistinct-0.6.4.tar.gz", hash = "sha256:ccd66ce4cdeefc20debe9e40a529de1d51f7175247a3216148d64fd29d3fe693"},+]++[package.dependencies]+m2r = "*"+numpy = "*"+scipy = "*"+statsmodels = "*"+xgboost = "*"+[[package]]name = "pyjwt"version = "2.8.0"@@ -2374,6 +2456,51 @@ anyio = ">=3.0.0,<4"[package.extras]full = ["graphene", "itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"]+[[package]]+name = "statsmodels"+version = "0.14.2"+description = "Statistical computations and models for Python"+optional = false+python-versions = ">=3.9"+files = [+ {file = "statsmodels-0.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df5d6f95c46f0341da6c79ee7617e025bf2b371e190a8e60af1ae9cabbdb7a97"},+ {file = "statsmodels-0.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a87ef21fadb445b650f327340dde703f13aec1540f3d497afb66324499dea97a"},+ {file = "statsmodels-0.14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5827a12e3ede2b98a784476d61d6bec43011fedb64aa815f2098e0573bece257"},+ {file = "statsmodels-0.14.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f2b7611a61adb7d596a6d239abdf1a4d5492b931b00d5ed23d32844d40e48e"},+ {file = "statsmodels-0.14.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c254c66142f1167b4c7d031cf8db55294cc62ff3280e090fc45bd10a7f5fd029"},+ {file = "statsmodels-0.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:0e46e9d59293c1af4cc1f4e5248f17e7e7bc596bfce44d327c789ac27f09111b"},+ {file = "statsmodels-0.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:50fcb633987779e795142f51ba49fb27648d46e8a1382b32ebe8e503aaabaa9e"},+ {file = "statsmodels-0.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:876794068abfaeed41df71b7887000031ecf44fbfa6b50d53ccb12ebb4ab747a"},+ {file = "statsmodels-0.14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a91f6c4943de13e3ce2e20ee3b5d26d02bd42300616a421becd53756f5deb37"},+ {file = "statsmodels-0.14.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4864a1c4615c5ea5f2e3b078a75bdedc90dd9da210a37e0738e064b419eccee2"},+ {file = "statsmodels-0.14.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:afbd92410e0df06f3d8c4e7c0e2e71f63f4969531f280fb66059e2ecdb6e0415"},+ {file = "statsmodels-0.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:8e004cfad0e46ce73fe3f3812010c746f0d4cfd48e307b45c14e9e360f3d2510"},+ {file = "statsmodels-0.14.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb0ba1ad3627705f5ae20af6b2982f500546d43892543b36c7bca3e2f87105e7"},+ {file = "statsmodels-0.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:90fd2f0110b73fc3fa5a2f21c3ca99b0e22285cccf38e56b5b8fd8ce28791b0f"},+ {file = "statsmodels-0.14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac780ad9ff552773798829a0b9c46820b0faa10e6454891f5e49a845123758ab"},+ {file = "statsmodels-0.14.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55d1742778400ae67acb04b50a2c7f5804182f8a874bd09ca397d69ed159a751"},+ {file = "statsmodels-0.14.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f870d14a587ea58a3b596aa994c2ed889cc051f9e450e887d2c83656fc6a64bf"},+ {file = "statsmodels-0.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:f450fcbae214aae66bd9d2b9af48e0f8ba1cb0e8596c6ebb34e6e3f0fec6542c"},+ {file = "statsmodels-0.14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:201c3d00929c4a67cda1fe05b098c8dcf1b1eeefa88e80a8f963a844801ed59f"},+ {file = "statsmodels-0.14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9edefa4ce08e40bc1d67d2f79bc686ee5e238e801312b5a029ee7786448c389a"},+ {file = "statsmodels-0.14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c78a7601fdae1aa32104c5ebff2e0b72c26f33e870e2f94ab1bcfd927ece9b"},+ {file = "statsmodels-0.14.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f36494df7c03d63168fccee5038a62f469469ed6a4dd6eaeb9338abedcd0d5f5"},+ {file = "statsmodels-0.14.2-cp39-cp39-win_amd64.whl", hash = "sha256:8875823bdd41806dc853333cc4e1b7ef9481bad2380a999e66ea42382cf2178d"},+ {file = "statsmodels-0.14.2.tar.gz", hash = "sha256:890550147ad3a81cda24f0ba1a5c4021adc1601080bd00e191ae7cd6feecd6ad"},+]++[package.dependencies]+numpy = ">=1.22.3"+packaging = ">=21.3"+pandas = ">=1.4,<2.1.0 || >2.1.0"+patsy = ">=0.5.6"+scipy = ">=1.8,<1.9.2 || >1.9.2"++[package.extras]+build = ["cython (>=0.29.33)"]+develop = ["colorama", "cython (>=0.29.33)", "cython (>=3.0.10,<4)", "flake8", "isort", "joblib", "matplotlib (>=3)", "pytest (>=7.3.0,<8)", "pytest-cov", "pytest-randomly", "pytest-xdist", "pywinpty", "setuptools-scm[toml] (>=8.0,<9.0)"]+docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"]+[[package]]name = "thinc"version = "8.0.17"@@ -2651,6 +2778,36 @@ files = [{file = "wasabi-0.10.1.tar.gz", hash = "sha256:c8e372781be19272942382b14d99314d175518d7822057cb7a97010c4259d249"},]+[[package]]+name = "xgboost"+version = "2.1.0"+description = "XGBoost Python Package"+optional = false+python-versions = ">=3.8"+files = [+ {file = "xgboost-2.1.0-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:19d145eb847b070c32342b1bf2d7331c102783e07a484f8b13b7d759d707c6b0"},+ {file = "xgboost-2.1.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:840a0c6e2119d8c8f260a5dace996ea064a267f62b301a25d7d452488a7ac860"},+ {file = "xgboost-2.1.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:73673c9bb85927db7fe2e3aed6df6d35dba708cfd6767cc63d4ea11dda2dede5"},+ {file = "xgboost-2.1.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:43b16205689249d7509daf7a6ab00ad0e6c570b3a9c263cb32b26e39d9477bb3"},+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:cedc2e386e686795735448fd4597533acacc5ba6fb47dd910c204c468b80bb96"},+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:b2a456eb0f3d3e8fd8ab37e44ac288292bf8ea8744c294be9fd88713d27af810"},+ {file = "xgboost-2.1.0-py3-none-win_amd64.whl", hash = "sha256:74904b91c42524a6c32147fe5718569e78fb65911ff4499b053f81d0964514d4"},+ {file = "xgboost-2.1.0.tar.gz", hash = "sha256:7144980923e76ce741c7b03a14d3bd7514db6de5c7cabe96ba95b229d274f5ca"},+]++[package.dependencies]+numpy = "*"+nvidia-nccl-cu12 = {version = "*", markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""}+scipy = "*"++[package.extras]+dask = ["dask", "distributed", "pandas"]+datatable = ["datatable"]+pandas = ["pandas"]+plotting = ["graphviz", "matplotlib"]+pyspark = ["cloudpickle", "pyspark", "scikit-learn"]+scikit-learn = ["scikit-learn"]+[[package]]name = "zstandard"version = "0.16.0"@@ -2716,4 +2873,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py[metadata]lock-version = "2.0"python-versions = ">=3.10,<3.11"-content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"+content-hash = "7ceca49e9a8b07803e268fe6048d27ed3803686a2c89d5aeaa64656ad71488f5"diff --git a/pyproject.toml b/pyproject.tomlindex 84dc93b..5aba154 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -46,6 +46,7 @@ lxml = "^5.2.0"numpy = "1.23.2"objgraph = "^3.6.1"psutil = "^6.0.0"+pydistinct = "^0.6.4"[tool.poetry.extras]indexer = ["ujson",