An open source, non-profit web search engine
Make lxml not optional
Daoud Clarkef7648e7d3374
diff --git a/poetry.lock b/poetry.lockindex 7b734d5..8df1356 100644--- a/poetry.lock+++ b/poetry.lock@@ -23,13 +23,13 @@ trio = ["trio (<0.22)"][[package]]name = "asgiref"-version = "3.7.2"+version = "3.8.1"description = "ASGI specs, helper code, and adapters"optional = false-python-versions = ">=3.7"+python-versions = ">=3.8"files = [- {file = "asgiref-3.7.2-py3-none-any.whl", hash = "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e"},- {file = "asgiref-3.7.2.tar.gz", hash = "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed"},+ {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"},+ {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"},][package.dependencies]@@ -134,17 +134,17 @@ numpy = {version = ">=1.19.0", markers = "python_version >= \"3.9\""}[[package]]name = "boto3"-version = "1.34.57"+version = "1.34.74"description = "The AWS SDK for Python"optional = false-python-versions = ">= 3.8"+python-versions = ">=3.8"files = [- {file = "boto3-1.34.57-py3-none-any.whl", hash = "sha256:f8046e3e2d1186a49b49f7464c4811c265c86001f404dd1a96c4365c773a4245"},- {file = "boto3-1.34.57.tar.gz", hash = "sha256:c26c31ceeeb2bc5d2bb96ba0fdc9a04d7b10e6e0b081c55b9cea9069a0be04dd"},+ {file = "boto3-1.34.74-py3-none-any.whl", hash = "sha256:71f551491fb12fe07727d371d5561c5919fdf33dbc1d4251c57940d267a53a9e"},+ {file = "boto3-1.34.74.tar.gz", hash = "sha256:b703e22775561a748adc4576c30424b81abd2a00d3c6fb28eec2e5cde92c1eed"},][package.dependencies]-botocore = ">=1.34.57,<1.35.0"+botocore = ">=1.34.74,<1.35.0"jmespath = ">=0.7.1,<2.0.0"s3transfer = ">=0.10.0,<0.11.0"@@ -153,19 +153,19 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"][[package]]name = "botocore"-version = "1.34.57"+version = "1.34.74"description = "Low-level, data-driven core of boto 3."optional = false-python-versions = ">= 3.8"+python-versions = ">=3.8"files = [- {file = "botocore-1.34.57-py3-none-any.whl", hash = "sha256:c8dafe0ad378a88bcf4153e6972870b03fb5aab406b694202307500709940baf"},- {file = "botocore-1.34.57.tar.gz", hash = "sha256:9a5aa2034de9f0c367b4b61a92af0fa827f5c21affa19e0a284838a142e71083"},+ {file = "botocore-1.34.74-py3-none-any.whl", hash = "sha256:5d2015b5d91d6c402c122783729ce995ed7283a746b0380957026dc2b3b75969"},+ {file = "botocore-1.34.74.tar.gz", hash = "sha256:32bb519bae62483893330c18a0ea4fd09d1ffa32bc573cd8559c2d9a08fb8c5c"},][package.dependencies]jmespath = ">=0.7.1,<2.0.0"python-dateutil = ">=2.1,<3.0.0"-urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}+urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}[package.extras]crt = ["awscrt (==0.19.19)"]@@ -1244,13 +1244,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"][[package]]name = "packaging"-version = "23.2"+version = "24.0"description = "Core utilities for Python packages"optional = falsepython-versions = ">=3.7"files = [- {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},- {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},+ {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},+ {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},][[package]]@@ -1559,13 +1559,13 @@ files = [[[package]]name = "pycparser"-version = "2.21"+version = "2.22"description = "C parser in Python"optional = false-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"+python-versions = ">=3.8"files = [- {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"},- {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},+ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},+ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},][[package]]@@ -1669,17 +1669,17 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no[[package]]name = "pytest-mock"-version = "3.12.0"+version = "3.14.0"description = "Thin-wrapper around the mock package for easier use with pytest"optional = falsepython-versions = ">=3.8"files = [- {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"},- {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"},+ {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"},+ {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"},][package.dependencies]-pytest = ">=5.0"+pytest = ">=6.2.5"[package.extras]dev = ["pre-commit", "pytest-asyncio", "tox"]@@ -1844,17 +1844,17 @@ full = ["numpy"][[package]]name = "redis"-version = "5.0.2"+version = "5.0.3"description = "Python client for Redis database and key-value store"optional = falsepython-versions = ">=3.7"files = [- {file = "redis-5.0.2-py3-none-any.whl", hash = "sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1"},- {file = "redis-5.0.2.tar.gz", hash = "sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037"},+ {file = "redis-5.0.3-py3-none-any.whl", hash = "sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d"},+ {file = "redis-5.0.3.tar.gz", hash = "sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580"},][package.dependencies]-async-timeout = ">=4.0.3"+async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""}hiredis = {version = ">=1.0.0", optional = true, markers = "extra == \"hiredis\""}[package.extras]@@ -1914,13 +1914,13 @@ yaml = ["pyyaml (>=6.0.1)"][[package]]name = "requests-oauthlib"-version = "1.3.1"+version = "2.0.0"description = "OAuthlib authentication support for Requests."optional = false-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"+python-versions = ">=3.4"files = [- {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"},- {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"},+ {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"},+ {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"},][package.dependencies]@@ -1932,13 +1932,13 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"][[package]]name = "s3transfer"-version = "0.10.0"+version = "0.10.1"description = "An Amazon S3 Transfer Manager"optional = falsepython-versions = ">= 3.8"files = [- {file = "s3transfer-0.10.0-py3-none-any.whl", hash = "sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e"},- {file = "s3transfer-0.10.0.tar.gz", hash = "sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b"},+ {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"},+ {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"},][package.dependencies]@@ -2033,18 +2033,18 @@ test = ["asv", "gmpy2", "hypothesis", "mpmath", "pooch", "pytest", "pytest-cov",[[package]]name = "setuptools"-version = "69.1.1"+version = "69.2.0"description = "Easily download, build, install, upgrade, and uninstall Python packages"optional = falsepython-versions = ">=3.8"files = [- {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"},- {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"},+ {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"},+ {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"},][package.extras]docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]-testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"][[package]]@@ -2338,13 +2338,13 @@ torch = ["torch (>=1.6.0)"][[package]]name = "threadpoolctl"-version = "3.3.0"+version = "3.4.0"description = "threadpoolctl"optional = falsepython-versions = ">=3.8"files = [- {file = "threadpoolctl-3.3.0-py3-none-any.whl", hash = "sha256:6155be1f4a39f31a18ea70f94a77e0ccd57dced08122ea61109e7da89883781e"},- {file = "threadpoolctl-3.3.0.tar.gz", hash = "sha256:5dac632b4fa2d43f42130267929af3ba01399ef4bd1882918e92dbc30365d30c"},+ {file = "threadpoolctl-3.4.0-py3-none-any.whl", hash = "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262"},+ {file = "threadpoolctl-3.4.0.tar.gz", hash = "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"},][[package]]@@ -2489,18 +2489,18 @@ six = "*"[[package]]name = "urllib3"-version = "2.0.7"+version = "2.2.1"description = "HTTP library with thread-safe connection pooling, file post, and more."optional = false-python-versions = ">=3.7"+python-versions = ">=3.8"files = [- {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"},- {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"},+ {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"},+ {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"},][package.extras]brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]-secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"]+h2 = ["h2 (>=4,<5)"]socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]zstd = ["zstandard (>=0.18.0)"]@@ -2613,4 +2613,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "lxml", "pyarr[metadata]lock-version = "2.0"python-versions = ">=3.10,<3.11"-content-hash = "f884b27d04703e4a611c6d37cc1b3153b9de0522637c3216672138ee6c22ffd6"+content-hash = "40a87fe1163206044a49cb7912b2c32038a7141888fa36360835d890f57888b0"diff --git a/pyproject.toml b/pyproject.tomlindex f24d8cd..8066b48 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -28,7 +28,7 @@ ujson = {version= "==4.3.0", optional = true}warcio = {version= "==1.7.4", optional = true}idna = {version= "==3.3", optional = true}beautifulsoup4 = {version= "==4.10.0", optional = true}-lxml = {version= "==4.6.4", optional = true}+lxml = {version= "==4.6.4"}langdetect = {version= "==1.0.9", optional = true}pyarrow = {version= "==6.0.0", optional = true}pyspark = {version= "==3.2.0", optional = true}