Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge branch 'main' into jwt-login

Daoud Clarke413d07cbb988
diff --cc mwmbl/crawler/app.py
index 547c4f3,6d51a86..cf96186
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@@ -3,7 -3,7 +3,6 @@@ import hashli
import json
import os
from datetime import datetime, timezone, date
--from queue import Queue, Empty
from typing import Union
from uuid import uuid4
@@@ -30,6 -34,7 +33,7 @@@ from mwmbl.settings import
PUBLIC_USER_ID_LENGTH,
FILE_NAME_SUFFIX,
DATE_REGEX)
-from mwmbl.tinysearchengine.indexer import Document, TinyIndex
++from mwmbl.tinysearchengine.indexer import Document
stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))
diff --cc poetry.lock
index 8bc6a49,ab0eade..1e10fd8
--- a/poetry.lock
+++ b/poetry.lock
@@@ -773,20 -836,24 +773,6 @@@ files =
{file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"},
]
--[[package]]
- name = "justext"
- version = "3.0.0"
- description = "Heuristic based boilerplate removal tool"
-name = "langcodes"
-version = "3.4.0"
-description = "Tools for labeling human languages with IETF language tags"
--optional = false
- python-versions = "*"
-python-versions = ">=3.8"
--files = [
- {file = "jusText-3.0.0-py2.py3-none-any.whl", hash = "sha256:86b48f5b1d99505acd072f5831def6cd3f1306043651c524a1c609e62e3544e4"},
- {file = "jusText-3.0.0.tar.gz", hash = "sha256:7640e248218795f6be65f6c35fe697325a3280fcb4675d1525bcdff2b86faadf"},
- {file = "langcodes-3.4.0-py3-none-any.whl", hash = "sha256:10a4cc078b8e8937d8485d3352312a0a89a3125190db9f2bb2074250eef654e9"},
- {file = "langcodes-3.4.0.tar.gz", hash = "sha256:ae5a77d1a01d0d1e91854a671890892b7ce9abb601ab7327fc5c874f899e1979"},
--]
--
--[package.dependencies]
- lxml = ">=4.4.2"
-language-data = ">=1.2"
-
-[package.extras]
-build = ["build", "twine"]
-test = ["pytest", "pytest-cov"]
--
[[package]]
name = "langdetect"
version = "1.0.9"
@@@ -2041,6 -2340,73 +2027,57 @@@ dev = ["cython-lint (>=0.12.2)", "doit
doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"]
test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
+ [[package]]
+ name = "sentry-sdk"
-version = "2.13.0"
++version = "2.14.0"
+ description = "Python client for Sentry (https://sentry.io)"
+ optional = false
+ python-versions = ">=3.6"
+ files = [
- {file = "sentry_sdk-2.13.0-py2.py3-none-any.whl", hash = "sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6"},
- {file = "sentry_sdk-2.13.0.tar.gz", hash = "sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260"},
++ {file = "sentry_sdk-2.14.0-py2.py3-none-any.whl", hash = "sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4"},
++ {file = "sentry_sdk-2.14.0.tar.gz", hash = "sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d"},
+ ]
+
+ [package.dependencies]
+ certifi = "*"
+ urllib3 = ">=1.26.11"
+
+ [package.extras]
+ aiohttp = ["aiohttp (>=3.5)"]
+ anthropic = ["anthropic (>=0.16)"]
+ arq = ["arq (>=0.23)"]
+ asyncpg = ["asyncpg (>=0.23)"]
+ beam = ["apache-beam (>=2.12)"]
+ bottle = ["bottle (>=0.12.13)"]
+ celery = ["celery (>=3)"]
+ celery-redbeat = ["celery-redbeat (>=2)"]
+ chalice = ["chalice (>=1.16.0)"]
+ clickhouse-driver = ["clickhouse-driver (>=0.2.0)"]
+ django = ["django (>=1.8)"]
+ falcon = ["falcon (>=1.4)"]
+ fastapi = ["fastapi (>=0.79.0)"]
+ flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"]
+ grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"]
+ httpx = ["httpx (>=0.16.0)"]
+ huey = ["huey (>=2)"]
+ huggingface-hub = ["huggingface-hub (>=0.22)"]
+ langchain = ["langchain (>=0.0.210)"]
+ litestar = ["litestar (>=2.0.0)"]
+ loguru = ["loguru (>=0.5)"]
+ openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"]
+ opentelemetry = ["opentelemetry-distro (>=0.35b0)"]
+ opentelemetry-experimental = ["opentelemetry-distro"]
+ pure-eval = ["asttokens", "executing", "pure-eval"]
+ pymongo = ["pymongo (>=3.1)"]
+ pyspark = ["pyspark (>=2.4.4)"]
+ quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]
+ rq = ["rq (>=0.6)"]
+ sanic = ["sanic (>=0.8)"]
+ sqlalchemy = ["sqlalchemy (>=1.2)"]
+ starlette = ["starlette (>=0.19.1)"]
+ starlite = ["starlite (>=1.48)"]
+ tornado = ["tornado (>=6)"]
+
-[[package]]
-name = "setuptools"
-version = "71.1.0"
-description = "Easily download, build, install, upgrade, and uninstall Python packages"
-optional = false
-python-versions = ">=3.8"
-files = [
- {file = "setuptools-71.1.0-py3-none-any.whl", hash = "sha256:33874fdc59b3188304b2e7c80d9029097ea31627180896fb549c578ceb8a0855"},
- {file = "setuptools-71.1.0.tar.gz", hash = "sha256:032d42ee9fb536e33087fb66cac5f840eb9391ed05637b3f2a76a7c8fb477936"},
-]
-
-[package.extras]
-core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"]
-doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
-test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
-
[[package]]
name = "six"
version = "1.16.0"
@@@ -2379,4 -3047,4 +2416,4 @@@ indexer = ["Levenshtein", "beautifulsou
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
- content-hash = "006391f32d00d91e320439a5324df3b543762ea5c7a09c83111c4a327f23a92b"
-content-hash = "de3cdafe7282c244e4670384daeb3315d13b9c374406a5d86ff8fada53bf75bf"
++content-hash = "f32650870abf870ad9031384c32c62d4ed076ec954ec70a3aaac90867bf170d0"
diff --cc pyproject.toml
index 96f4888,8b7f7cb..ec1f642
--- a/pyproject.toml
+++ b/pyproject.toml
@@@ -16,9 -17,9 +16,8 @@@ pyyaml = "==6.0
boto3 = "^1.20.37"
requests = "^2.27.1"
psycopg2-binary = "^2.9.3"
-spacy = "==3.2.1"
pytest = "^7.2.1"
pytest-mock = "^3.10.0"
- jusText = "==3.0.0"
# Optional dependencies do not get installed by default. Look under tool.poetry.extras section
# to see which extras to use.
@@@ -46,7 -47,7 +45,8 @@@ objgraph = "^3.6.1
psutil = "^6.0.0"
xgboost = "^2.1.0"
pydistinct = "^0.6.4"
+django-ninja-jwt = "^5.3.3"
+ sentry-sdk = "^2.13.0"
[tool.poetry.extras]
indexer = [