Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Estimate counts instead of using a hyperloglog

Daoud Clarked2448ae10130
diff --git a/analyse/estimate_unique_urls.py b/analyse/estimate_unique_urls.py
index ec6d821..dd38c5f 100644
--- a/analyse/estimate_unique_urls.py
+++ b/analyse/estimate_unique_urls.py
@@ -7,8 +7,11 @@ from pathlib import Path
from random import Random
import numpy as np
+from pydistinct.ensemble_estimators import median_estimator
+from pydistinct.stats_estimators import bootstrap_estimator, goodmans_estimator, smoothed_jackknife_estimator, \
+ horvitz_thompson_estimator
from scipy.optimize import curve_fit, differential_evolution
-from scipy.stats import binom, poisson
+from scipy.stats import binom, poisson, betabinom
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -41,58 +44,11 @@ def estimate_unique_urls(index_path: str, num_pages_to_sample: int = 100):
url_counts.update({doc.url for doc in page})
total_docs += len(page)
- frequencies = Counter(url_counts.values())
- print("Frequencies", frequencies)
-
- # Fit a binomial distribution to the data
- def binomial(x, p, s):
- print("Binomial", x, p, s)
- return binom(num_pages_to_sample, p).pmf(x) * s
-
- frequencies = dict(sorted(frequencies.items()))
- freq = np.array(list(frequencies.keys()))
- values = np.array(list(frequencies.values()))
-
- def poiss(x, m, s):
- return poisson.pmf(x, m) * s
-
- m_fit, s_fit = curve_fit(poiss, freq, values)[0]
- print("Estimated parameter m", m_fit, s_fit)
-
- predictions = poiss(freq, m_fit, s_fit)
- print("Predictions", predictions)
- print("Actual", values)
-
-
- #
- # def cost_function(x):
- # p, s = x
- # value = np.sum((binomial(freq, p, s) - values) ** 2)
- # return value
- #
- # result = differential_evolution(cost_function, bounds=[(0.0, 1.0), (1.0, 1e100)], args=(), maxiter=100_000, strategy='rand2bin', tol=0.001)
- # print("Result", result)
- #
- # p, s = result.x
- # print("Predicted values", binomial(freq, p, s))
- # print("Actual", values)
-
- # num_unique_urls = len(url_counts)
- # mean_results_per_page = total_docs / num_pages_to_sample
- # initial_p = mean_results_per_page / num_unique_urls
- # print("Initial p", initial_p)
- # print("Initial s", num_unique_urls)
- # p0 = [initial_p, num_unique_urls]
- # f0 = binomial(freq, *p0)
- # print("F0", f0)
- #
- # p_fit, s_fit = curve_fit(binomial, freq, values, bounds=[(0.0, 1.0), (1.0, 1e100)],
- # p0=p0)[0]
- # print("Estimated parameters p, s", p_fit, s_fit)
+ return median_estimator(attributes=dict(url_counts.items()))
if __name__ == "__main__":
- estimate = estimate_unique_urls(str(DEV_INDEX_PATH), 1000)
+ estimate = estimate_unique_urls(str(DEV_INDEX_PATH), 500)
num_urls = count_unique_urls(str(DEV_INDEX_PATH))
print(f"Actual number of unique URLs: {num_urls}")
diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.py
index 57d8afa..fd0fb02 100644
--- a/mwmbl/count_urls.py
+++ b/mwmbl/count_urls.py
@@ -2,29 +2,29 @@ import os
from collections import Counter
from datetime import date, timedelta, datetime
from logging import getLogger
-from multiprocessing import Pool
from pathlib import Path
+from random import Random
from time import sleep
from django.conf import settings
+from pydistinct.ensemble_estimators import median_estimator
from redis import Redis
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
-from mwmbl.utils import batch, parse_url
+from mwmbl.utils import parse_url
INDEX_RESULT_COUNT_KEY = "index-result-count-{date}"
INDEX_DOMAIN_COUNT_KEY = "index-domain-count-{date}"
INDEX_URL_COUNT_KEY = "index-url-count-{date}"
INDEX_DOMAIN_RESULT_COUNT_KEY = "index-domain-result-count-{date}"
-INDEX_URL_HLL_KEY = "index-hll-{date}"
-INDEX_DOMAIN_HLL_KEY = "index-domain-hll-{date}"
-SHORT_EXPIRE_SECONDS = 60 * 60 * 24
LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30
-NUM_PAGES_IN_BATCH = 10240
+
+PAGE_PROPORTION_TO_SAMPLE = 0.0001
logger = getLogger(__name__)
+random = Random(1)
def get_redis():
@@ -43,75 +43,44 @@ def count_urls_continuously():
def count_urls():
- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:
- page_indexes = batch(range(tiny_index.num_pages), NUM_PAGES_IN_BATCH)
-
start_time = datetime.utcnow()
- with Pool(processes=4) as pool:
- num_results = sum(pool.map(count_urls_in_batch, page_indexes))
- logger.info(f"Processed {num_results} results in total.")
-
- domain_count_key, domain_hll_key, url_hll_key = _get_keys()
+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
+ with TinyIndex(item_factory=Document, index_path=index_path) as index:
+ page_sample = set()
+ num_pages_to_sample = max(100, int(index.num_pages * PAGE_PROPORTION_TO_SAMPLE))
+ logger.info(f"Sampling {num_pages_to_sample} pages.")
+ while len(page_sample) < num_pages_to_sample:
+ page_sample.add(random.randrange(index.num_pages))
+
+ url_counts = Counter()
+ domain_counts = Counter()
+ total_docs = 0
+ for i in page_sample:
+ page = index.get_page(i)
+ url_counts.update({doc.url for doc in page})
+ domains = [parse_url(doc.url).netloc for doc in page]
+ domain_counts.update(domains)
+ total_docs += len(page)
+
+ url_count_estimate = median_estimator(attributes=dict(url_counts.items()))
+ domain_count_estimate = median_estimator(attributes=dict(domain_counts.items()))
+ num_results_estimate = total_docs / PAGE_PROPORTION_TO_SAMPLE
+
+ logger.info(f"Estimated {url_count_estimate} unique URLs, {domain_count_estimate} unique domains, "
+ f"and {num_results_estimate} results in the index.")
redis = get_redis()
- redis.expire(url_hll_key, SHORT_EXPIRE_SECONDS)
- url_count = redis.pfcount(url_hll_key)
- logger.info("Counted %d unique URLs", url_count)
-
- redis.expire(domain_hll_key, SHORT_EXPIRE_SECONDS)
- domain_count = redis.pfcount(domain_hll_key)
- logger.info("Counted %d unique domains", domain_count)
-
- redis.expire(domain_count_key, SHORT_EXPIRE_SECONDS)
today = date.today()
- _set_count(INDEX_URL_COUNT_KEY, redis, today, url_count)
- _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, domain_count)
- _set_count(INDEX_RESULT_COUNT_KEY, redis, today, num_results)
+ _set_count(INDEX_URL_COUNT_KEY, redis, today, int(url_count_estimate))
+ _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, int(domain_count_estimate))
+ _set_count(INDEX_RESULT_COUNT_KEY, redis, today, int(num_results_estimate))
end_time = datetime.utcnow()
logger.info(f"Counting took {end_time - start_time}.")
-def count_urls_in_batch(page_indexes: list[int]):
- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
- redis = get_redis()
-
- logger.info(f"Counting URLs in index {index_path}")
-
- domain_count_key, domain_hll_key, url_hll_key = _get_keys()
- num_results = 0
- with TinyIndex(item_factory=Document, index_path=index_path) as tiny_index:
- urls = set()
- domains = set()
- domain_counts = Counter()
- for i in page_indexes:
- docs = tiny_index.get_page(i)
- urls |= {doc.url for doc in docs}
- page_domains = [parse_url(doc.url).netloc for doc in docs]
- domain_counts.update(page_domains)
- domains |= set(page_domains)
- num_results += len(docs)
-
- for domain, count in domain_counts.items():
- redis.zincrby(domain_count_key, count, domain)
-
- redis.pfadd(url_hll_key, *urls)
- redis.pfadd(domain_hll_key, *domains)
- logger.info(f"Counted {i} pages of {tiny_index.num_pages}.")
- return num_results
-
-
-def _get_keys():
- today = date.today()
- url_hll_key = INDEX_URL_HLL_KEY.format(date=today)
- domain_hll_key = INDEX_DOMAIN_HLL_KEY.format(date=today)
- domain_count_key = INDEX_DOMAIN_RESULT_COUNT_KEY.format(date=today)
- return domain_count_key, domain_hll_key, url_hll_key
-
-
def _set_count(key, redis, today, count):
redis.set(key.format(date=today), count)
redis.expire(key.format(date=today), LONG_EXPIRE_SECONDS)
diff --git a/poetry.lock b/poetry.lock
index 2e44154..68bbe78 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -620,6 +620,17 @@ Django = ">=1.11"
[package.extras]
dev = ["black", "flake8"]
+[[package]]
+name = "docutils"
+version = "0.21.2"
+description = "Docutils -- Python Documentation Utilities"
+optional = false
+python-versions = ">=3.9"
+files = [
+ {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"},
+ {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"},
+]
+
[[package]]
name = "exceptiongroup"
version = "1.2.0"
@@ -1094,6 +1105,20 @@ files = [
[package.dependencies]
lxml = "*"
+[[package]]
+name = "m2r"
+version = "0.3.1"
+description = "Markdown and reStructuredText in a single file."
+optional = false
+python-versions = "*"
+files = [
+ {file = "m2r-0.3.1.tar.gz", hash = "sha256:aafb67fc49cfb1d89e46a3443ac747e15f4bb42df20ed04f067ad9efbee256ab"},
+]
+
+[package.dependencies]
+docutils = "*"
+mistune = "<2"
+
[[package]]
name = "markupsafe"
version = "2.1.5"
@@ -1163,6 +1188,17 @@ files = [
{file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"},
]
+[[package]]
+name = "mistune"
+version = "0.8.4"
+description = "The fastest markdown parser in pure Python"
+optional = false
+python-versions = "*"
+files = [
+ {file = "mistune-0.8.4-py2.py3-none-any.whl", hash = "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4"},
+ {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"},
+]
+
[[package]]
name = "mmh3"
version = "3.1.0"
@@ -1286,6 +1322,16 @@ files = [
{file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"},
]
+[[package]]
+name = "nvidia-nccl-cu12"
+version = "2.22.3"
+description = "NVIDIA Collective Communication Library (NCCL) Runtime"
+optional = false
+python-versions = ">=3"
+files = [
+ {file = "nvidia_nccl_cu12-2.22.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9f5e03c00269dee2cd1aa57019f9a024478a74ae6e9b32d5341c849fe6f6302"},
+]
+
[[package]]
name = "oauthlib"
version = "3.2.2"
@@ -1405,6 +1451,24 @@ gcs = ["google-cloud-storage (>=1.26.0,<2.0.0)"]
s3 = ["boto3"]
test = ["mock", "pytest", "pytest-coverage", "typer-cli"]
+[[package]]
+name = "patsy"
+version = "0.5.6"
+description = "A Python package for describing statistical models and for building design matrices."
+optional = false
+python-versions = "*"
+files = [
+ {file = "patsy-0.5.6-py2.py3-none-any.whl", hash = "sha256:19056886fd8fa71863fa32f0eb090267f21fb74be00f19f5c70b2e9d76c883c6"},
+ {file = "patsy-0.5.6.tar.gz", hash = "sha256:95c6d47a7222535f84bff7f63d7303f2e297747a598db89cf5c67f0c0c7d2cdb"},
+]
+
+[package.dependencies]
+numpy = ">=1.4"
+six = "*"
+
+[package.extras]
+test = ["pytest", "pytest-cov", "scipy"]
+
[[package]]
name = "platformdirs"
version = "4.2.0"
@@ -1709,6 +1773,24 @@ typing-extensions = ">=3.7.4.3"
dotenv = ["python-dotenv (>=0.10.4)"]
email = ["email-validator (>=1.0.3)"]
+[[package]]
+name = "pydistinct"
+version = "0.6.4"
+description = "Package for estimating distinct values in a population"
+optional = false
+python-versions = "*"
+files = [
+ {file = "pydistinct-0.6.4-py3-none-any.whl", hash = "sha256:c5089aa0cad9b8bbf26e2735cd5cdd69289a8d3ce82e9b740335feab5a3ce149"},
+ {file = "pydistinct-0.6.4.tar.gz", hash = "sha256:ccd66ce4cdeefc20debe9e40a529de1d51f7175247a3216148d64fd29d3fe693"},
+]
+
+[package.dependencies]
+m2r = "*"
+numpy = "*"
+scipy = "*"
+statsmodels = "*"
+xgboost = "*"
+
[[package]]
name = "pyjwt"
version = "2.8.0"
@@ -2374,6 +2456,51 @@ anyio = ">=3.0.0,<4"
[package.extras]
full = ["graphene", "itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"]
+[[package]]
+name = "statsmodels"
+version = "0.14.2"
+description = "Statistical computations and models for Python"
+optional = false
+python-versions = ">=3.9"
+files = [
+ {file = "statsmodels-0.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df5d6f95c46f0341da6c79ee7617e025bf2b371e190a8e60af1ae9cabbdb7a97"},
+ {file = "statsmodels-0.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a87ef21fadb445b650f327340dde703f13aec1540f3d497afb66324499dea97a"},
+ {file = "statsmodels-0.14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5827a12e3ede2b98a784476d61d6bec43011fedb64aa815f2098e0573bece257"},
+ {file = "statsmodels-0.14.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f2b7611a61adb7d596a6d239abdf1a4d5492b931b00d5ed23d32844d40e48e"},
+ {file = "statsmodels-0.14.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c254c66142f1167b4c7d031cf8db55294cc62ff3280e090fc45bd10a7f5fd029"},
+ {file = "statsmodels-0.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:0e46e9d59293c1af4cc1f4e5248f17e7e7bc596bfce44d327c789ac27f09111b"},
+ {file = "statsmodels-0.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:50fcb633987779e795142f51ba49fb27648d46e8a1382b32ebe8e503aaabaa9e"},
+ {file = "statsmodels-0.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:876794068abfaeed41df71b7887000031ecf44fbfa6b50d53ccb12ebb4ab747a"},
+ {file = "statsmodels-0.14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a91f6c4943de13e3ce2e20ee3b5d26d02bd42300616a421becd53756f5deb37"},
+ {file = "statsmodels-0.14.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4864a1c4615c5ea5f2e3b078a75bdedc90dd9da210a37e0738e064b419eccee2"},
+ {file = "statsmodels-0.14.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:afbd92410e0df06f3d8c4e7c0e2e71f63f4969531f280fb66059e2ecdb6e0415"},
+ {file = "statsmodels-0.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:8e004cfad0e46ce73fe3f3812010c746f0d4cfd48e307b45c14e9e360f3d2510"},
+ {file = "statsmodels-0.14.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb0ba1ad3627705f5ae20af6b2982f500546d43892543b36c7bca3e2f87105e7"},
+ {file = "statsmodels-0.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:90fd2f0110b73fc3fa5a2f21c3ca99b0e22285cccf38e56b5b8fd8ce28791b0f"},
+ {file = "statsmodels-0.14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac780ad9ff552773798829a0b9c46820b0faa10e6454891f5e49a845123758ab"},
+ {file = "statsmodels-0.14.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55d1742778400ae67acb04b50a2c7f5804182f8a874bd09ca397d69ed159a751"},
+ {file = "statsmodels-0.14.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f870d14a587ea58a3b596aa994c2ed889cc051f9e450e887d2c83656fc6a64bf"},
+ {file = "statsmodels-0.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:f450fcbae214aae66bd9d2b9af48e0f8ba1cb0e8596c6ebb34e6e3f0fec6542c"},
+ {file = "statsmodels-0.14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:201c3d00929c4a67cda1fe05b098c8dcf1b1eeefa88e80a8f963a844801ed59f"},
+ {file = "statsmodels-0.14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9edefa4ce08e40bc1d67d2f79bc686ee5e238e801312b5a029ee7786448c389a"},
+ {file = "statsmodels-0.14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c78a7601fdae1aa32104c5ebff2e0b72c26f33e870e2f94ab1bcfd927ece9b"},
+ {file = "statsmodels-0.14.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f36494df7c03d63168fccee5038a62f469469ed6a4dd6eaeb9338abedcd0d5f5"},
+ {file = "statsmodels-0.14.2-cp39-cp39-win_amd64.whl", hash = "sha256:8875823bdd41806dc853333cc4e1b7ef9481bad2380a999e66ea42382cf2178d"},
+ {file = "statsmodels-0.14.2.tar.gz", hash = "sha256:890550147ad3a81cda24f0ba1a5c4021adc1601080bd00e191ae7cd6feecd6ad"},
+]
+
+[package.dependencies]
+numpy = ">=1.22.3"
+packaging = ">=21.3"
+pandas = ">=1.4,<2.1.0 || >2.1.0"
+patsy = ">=0.5.6"
+scipy = ">=1.8,<1.9.2 || >1.9.2"
+
+[package.extras]
+build = ["cython (>=0.29.33)"]
+develop = ["colorama", "cython (>=0.29.33)", "cython (>=3.0.10,<4)", "flake8", "isort", "joblib", "matplotlib (>=3)", "pytest (>=7.3.0,<8)", "pytest-cov", "pytest-randomly", "pytest-xdist", "pywinpty", "setuptools-scm[toml] (>=8.0,<9.0)"]
+docs = ["ipykernel", "jupyter-client", "matplotlib", "nbconvert", "nbformat", "numpydoc", "pandas-datareader", "sphinx"]
+
[[package]]
name = "thinc"
version = "8.0.17"
@@ -2651,6 +2778,36 @@ files = [
{file = "wasabi-0.10.1.tar.gz", hash = "sha256:c8e372781be19272942382b14d99314d175518d7822057cb7a97010c4259d249"},
]
+[[package]]
+name = "xgboost"
+version = "2.1.0"
+description = "XGBoost Python Package"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "xgboost-2.1.0-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:19d145eb847b070c32342b1bf2d7331c102783e07a484f8b13b7d759d707c6b0"},
+ {file = "xgboost-2.1.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:840a0c6e2119d8c8f260a5dace996ea064a267f62b301a25d7d452488a7ac860"},
+ {file = "xgboost-2.1.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:73673c9bb85927db7fe2e3aed6df6d35dba708cfd6767cc63d4ea11dda2dede5"},
+ {file = "xgboost-2.1.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:43b16205689249d7509daf7a6ab00ad0e6c570b3a9c263cb32b26e39d9477bb3"},
+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:cedc2e386e686795735448fd4597533acacc5ba6fb47dd910c204c468b80bb96"},
+ {file = "xgboost-2.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:b2a456eb0f3d3e8fd8ab37e44ac288292bf8ea8744c294be9fd88713d27af810"},
+ {file = "xgboost-2.1.0-py3-none-win_amd64.whl", hash = "sha256:74904b91c42524a6c32147fe5718569e78fb65911ff4499b053f81d0964514d4"},
+ {file = "xgboost-2.1.0.tar.gz", hash = "sha256:7144980923e76ce741c7b03a14d3bd7514db6de5c7cabe96ba95b229d274f5ca"},
+]
+
+[package.dependencies]
+numpy = "*"
+nvidia-nccl-cu12 = {version = "*", markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""}
+scipy = "*"
+
+[package.extras]
+dask = ["dask", "distributed", "pandas"]
+datatable = ["datatable"]
+pandas = ["pandas"]
+plotting = ["graphviz", "matplotlib"]
+pyspark = ["cloudpickle", "pyspark", "scikit-learn"]
+scikit-learn = ["scikit-learn"]
+
[[package]]
name = "zstandard"
version = "0.16.0"
@@ -2716,4 +2873,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
-content-hash = "45fc86862c2dba353d6bd23111d063993023cef0f77474371333902d92b8e312"
+content-hash = "7ceca49e9a8b07803e268fe6048d27ed3803686a2c89d5aeaa64656ad71488f5"
diff --git a/pyproject.toml b/pyproject.toml
index 84dc93b..5aba154 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,6 +46,7 @@ lxml = "^5.2.0"
numpy = "1.23.2"
objgraph = "^3.6.1"
psutil = "^6.0.0"
+pydistinct = "^0.6.4"
[tool.poetry.extras]
indexer = [
"ujson",