An open source, non-profit web search engine
Merge branch 'main' into jwt-login
Daoud Clarke413d07cbb988
diff --cc mwmbl/crawler/app.pyindex 547c4f3,6d51a86..cf96186--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@@ -3,7 -3,7 +3,6 @@@ import hashliimport jsonimport osfrom datetime import datetime, timezone, date--from queue import Queue, Emptyfrom typing import Unionfrom uuid import uuid4@@@ -30,6 -34,7 +33,7 @@@ from mwmbl.settings importPUBLIC_USER_ID_LENGTH,FILE_NAME_SUFFIX,DATE_REGEX)-from mwmbl.tinysearchengine.indexer import Document, TinyIndex++from mwmbl.tinysearchengine.indexer import Documentstats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))diff --cc poetry.lockindex 8bc6a49,ab0eade..1e10fd8--- a/poetry.lock+++ b/poetry.lock@@@ -773,20 -836,24 +773,6 @@@ files ={file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},]--[[package]]- name = "justext"- version = "3.0.0"- description = "Heuristic based boilerplate removal tool"-name = "langcodes"-version = "3.4.0"-description = "Tools for labeling human languages with IETF language tags"--optional = false- python-versions = "*"-python-versions = ">=3.8"--files = [- {file = "jusText-3.0.0-py2.py3-none-any.whl", hash = "sha256:86b48f5b1d99505acd072f5831def6cd3f1306043651c524a1c609e62e3544e4"},- {file = "jusText-3.0.0.tar.gz", hash = "sha256:7640e248218795f6be65f6c35fe697325a3280fcb4675d1525bcdff2b86faadf"},- {file = "langcodes-3.4.0-py3-none-any.whl", hash = "sha256:10a4cc078b8e8937d8485d3352312a0a89a3125190db9f2bb2074250eef654e9"},- {file = "langcodes-3.4.0.tar.gz", hash = "sha256:ae5a77d1a01d0d1e91854a671890892b7ce9abb601ab7327fc5c874f899e1979"},--]----[package.dependencies]- lxml = ">=4.4.2"-language-data = ">=1.2"--[package.extras]-build = ["build", "twine"]-test = ["pytest", "pytest-cov"]--[[package]]name = "langdetect"version = "1.0.9"@@@ -2041,6 -2340,73 +2027,57 @@@ dev = ["cython-lint (>=0.12.2)", "doitdoc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"]test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]+ [[package]]+ name = "sentry-sdk"-version = "2.13.0"++version = "2.14.0"+ description = "Python client for Sentry (https://sentry.io)"+ optional = false+ python-versions = ">=3.6"+ files = [- {file = "sentry_sdk-2.13.0-py2.py3-none-any.whl", hash = "sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6"},- {file = "sentry_sdk-2.13.0.tar.gz", hash = "sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260"},++ {file = "sentry_sdk-2.14.0-py2.py3-none-any.whl", hash = "sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4"},++ {file = "sentry_sdk-2.14.0.tar.gz", hash = "sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d"},+ ]++ [package.dependencies]+ certifi = "*"+ urllib3 = ">=1.26.11"++ [package.extras]+ aiohttp = ["aiohttp (>=3.5)"]+ anthropic = ["anthropic (>=0.16)"]+ arq = ["arq (>=0.23)"]+ asyncpg = ["asyncpg (>=0.23)"]+ beam = ["apache-beam (>=2.12)"]+ bottle = ["bottle (>=0.12.13)"]+ celery = ["celery (>=3)"]+ celery-redbeat = ["celery-redbeat (>=2)"]+ chalice = ["chalice (>=1.16.0)"]+ clickhouse-driver = ["clickhouse-driver (>=0.2.0)"]+ django = ["django (>=1.8)"]+ falcon = ["falcon (>=1.4)"]+ fastapi = ["fastapi (>=0.79.0)"]+ flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"]+ grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"]+ httpx = ["httpx (>=0.16.0)"]+ huey = ["huey (>=2)"]+ huggingface-hub = ["huggingface-hub (>=0.22)"]+ langchain = ["langchain (>=0.0.210)"]+ litestar = ["litestar (>=2.0.0)"]+ loguru = ["loguru (>=0.5)"]+ openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"]+ opentelemetry = ["opentelemetry-distro (>=0.35b0)"]+ opentelemetry-experimental = ["opentelemetry-distro"]+ pure-eval = ["asttokens", "executing", "pure-eval"]+ pymongo = ["pymongo (>=3.1)"]+ pyspark = ["pyspark (>=2.4.4)"]+ quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]+ rq = ["rq (>=0.6)"]+ sanic = ["sanic (>=0.8)"]+ sqlalchemy = ["sqlalchemy (>=1.2)"]+ starlette = ["starlette (>=0.19.1)"]+ starlite = ["starlite (>=1.48)"]+ tornado = ["tornado (>=6)"]+-[[package]]-name = "setuptools"-version = "71.1.0"-description = "Easily download, build, install, upgrade, and uninstall Python packages"-optional = false-python-versions = ">=3.8"-files = [- {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"},- {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"},-]--[package.extras]-core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"]-doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]-test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]-[[package]]name = "six"version = "1.16.0"@@@ -2379,4 -3047,4 +2416,4 @@@ indexer = ["Levenshtein", "beautifulsou[metadata]lock-version = "2.0"python-versions = ">=3.10,<3.11"- content-hash = "006391f32d00d91e320439a5324df3b543762ea5c7a09c83111c4a327f23a92b"-content-hash = "de3cdafe7282c244e4670384daeb3315d13b9c374406a5d86ff8fada53bf75bf"++content-hash = "f32650870abf870ad9031384c32c62d4ed076ec954ec70a3aaac90867bf170d0"diff --cc pyproject.tomlindex 96f4888,8b7f7cb..ec1f642--- a/pyproject.toml+++ b/pyproject.toml@@@ -16,9 -17,9 +16,8 @@@ pyyaml = "==6.0boto3 = "^1.20.37"requests = "^2.27.1"psycopg2-binary = "^2.9.3"-spacy = "==3.2.1"pytest = "^7.2.1"pytest-mock = "^3.10.0"- jusText = "==3.0.0"# Optional dependencies do not get installed by default. Look under tool.poetry.extras section# to see which extras to use.@@@ -46,7 -47,7 +45,8 @@@ objgraph = "^3.6.1psutil = "^6.0.0"xgboost = "^2.1.0"pydistinct = "^0.6.4"+django-ninja-jwt = "^5.3.3"+ sentry-sdk = "^2.13.0"[tool.poetry.extras]indexer = [