An open source, non-profit web search engine
[build-system]
requires = ["maturin>=1.5,<2.0", "patchelf"]
build-backend = "maturin"
# PEP 621 project metadata
[project]
name = "mwmbl"
version = "0.1.0"
description = ""
requires-python = ">=3.11,<3.12"
dependencies = [
"pandas>=1.3.5,<2.0.0",
"scipy>=1.8.0",
"scikit-learn>=1.0.2",
"zstandard>=0.16.0",
"mmh3>=3.0.0",
"uvicorn>=0.16.0",
"gunicorn>=21.0.0",
"pyyaml==6.0",
"boto3==1.35.99",
"requests>=2.27.1",
"psycopg2-binary>=2.9.3",
"django>=4.2.4",
"django-ninja==1.3.0",
"requests-cache>=1.1.0",
"redis[hiredis]>=5.0.1",
"django-allauth==0.57.2",
"dj-database-url>=2.1.0",
"django-htmx>=1.17.0",
"django-vite>=2.1.3",
"pybloomfiltermmap3>=0.6.0",
"lxml-html-clean>=0.1.0",
"lxml>=5.2.0",
"numpy==1.23.2",
"objgraph>=3.6.1",
"psutil>=6.0.0",
# xgboost's Linux wheels bundle a CUDA-enabled libxgboost.so and pull in
# nvidia-nccl-cu12, together ~600MB installed. Since 3.1.3 that applies to
# arm64 as well, which is what doubled the crawler image. Nothing here trains
# or predicts on a GPU - serving goes through the Rust extension's own
# libxgboost - so take the CPU-only build wherever it is published (Linux and
# Windows). The macOS wheel has no CUDA in it to begin with.
"xgboost-cpu>=2.1.0; sys_platform != 'darwin'",
"xgboost>=2.1.0; sys_platform == 'darwin'",
"pydistinct>=0.6.4",
"django-ninja-jwt>=5.3.3",
"sentry-sdk>=2.13.0",
"django-debug-toolbar>=5.2.0",
"wasmtime>=24.0.0",
"dotenv>=0.9.9",
"py-spy>=0.4.1",
"scalar-ninja>=0.3.0",
"django-background-tasks>=1.2.8",
"django-redis>=5.4.0",
"polar-sdk==0.31.7",
"httpx>=0.27.0",
"defusedxml>=0.7.0",
# Used by the Super Search recipe engine (super_search_sources/recipe.py),
# which is imported at app startup — must be a core dep, not indexer-only.
"beautifulsoup4==4.10.0",
"onnxruntime>=1.27.0",
"tokenizers>=0.23.1",
"standardwebhooks>=1.0.1",
]
[project.optional-dependencies]
indexer = [
"ujson==4.3.0",
"warcio==1.7.4",
"idna==3.3",
"langdetect==1.0.9",
"pyarrow==6.0.0",
"pyspark==3.2.0",
"Levenshtein==0.16.0",
]
[project.scripts]
mwmbl-tinysearchengine = "mwmbl.main:run"
mwmbl-crawl = "mwmbl.crawl:main"
[dependency-groups]
dev = [
"pytest>=7.0.1",
"pytest-mock>=3.10.0",
"pytest-django>=4.5.2",
"pytest-asyncio>=0.23.0",
"pytest-httpx>=0.30.0",
"pytest-cov>=5.0.0",
"blockbuster>=1.5.0",
"fakeredis>=2.30.1",
"maturin>=1.5,<2.0",
"patchelf>=0.17.2.4",
"ruff>=0.16.6",
"ty>=0.0.78",
"pre-commit>=4.6.2",
]
[tool.uv]
# pydistinct declares xgboost as a dependency but never imports it - its four modules
# use numpy, scipy and statsmodels only. On Linux that stale requirement is enough to
# drag the CUDA wheel and nvidia-nccl-cu12 back in, undoing the xgboost-cpu switch
# above, so confine every xgboost request to macOS where the wheel is CPU-only anyway.
override-dependencies = ["xgboost>=2.1.0; sys_platform == 'darwin'"]
[tool.maturin]
python-source = "."
module-name = "mwmbl_rank"
manifest-path = "mwmbl_rank/Cargo.toml"
# Include the mwmbl Python package and the generated domains JSON
python-packages = ["mwmbl"]
include = ["mwmbl_rank/data/hn_top_domains_filtered.json"]
[tool.pytest.ini_options]
log_cli = true
log_cli_level = "INFO"
DJANGO_SETTINGS_MODULE = "mwmbl.settings_test"
python_files = ["tests.py", "test_*.py", "*_tests.py"]
addopts = "--reuse-db -m 'not live and not slow'"
asyncio_mode = "auto"
# DTZ cannot see this one: at the point of assignment `timezone.now()` and
# `datetime.utcnow()` look the same to a linter, and only Django knows the column is aware.
filterwarnings = ["error:.*received a naive datetime:RuntimeWarning"]
markers = [
"live: hits real external endpoints; excluded from the default run. Run with -m live (used by the weekly Super Search smoke workflow).",
"slow: heavy CPU (e.g. XGBoost feature selection); excluded from the default run. Run with -m slow.",
]
[tool.coverage.run]
source = ["mwmbl"]
omit = [
"mwmbl/migrations/*",
"mwmbl/asgi.py",
"mwmbl/wsgi.py",
"mwmbl/indexer/paths.py",
]
[tool.coverage.report]
exclude_also = [
"if TYPE_CHECKING:",
"@(abc\\.)?abstractmethod",
]
[tool.ruff]
line-length = 120
target-version = "py311"
extend-exclude = ["devdata", "front-end", "test_env"]
[tool.ruff.lint]
select = ["DTZ", "E", "F", "I"]
# The formatter owns line length. E501 only ever fires on things it cannot split -
# long string literals, URLs and markdown tables in API docs - where wrapping hurts.
ignore = ["E501"]
[tool.ruff.lint.per-file-ignores]
# Django settings modules build their config with star imports from a base module.
"mwmbl/settings*.py" = ["F403", "F405"]
# These modules must call django.setup() (or pytest.importorskip) before the imports
# that reach Django models or the Rust extension, so those imports cannot be at the top.
"analyse/analyse_crawled_links.py" = ["E402"]
"mwmbl/crawl.py" = ["E402"]
"mwmbl/rankeval/evaluation/compare_ltr_models.py" = ["E402"]
"test/test_rust_features.py" = ["E402"]
"scripts/combined_search_haiku/*.py" = ["E402"]
[tool.ruff.lint.isort]
known-first-party = ["mwmbl", "mwmbl_rank"]
[tool.ty.environment]
python-version = "3.11"
[tool.ty.src]
exclude = ["devdata", "front-end", "test_env", "scripts/train", "**/migrations"]
# ty is gated on CI and on commit, so every rule that has pre-existing violations is
# downgraded to "warn" here: the baseline is green and a NEW category of error fails the
# build. `make typecheck-warnings` lists everything still outstanding.
#
# Tighten these back to "error" one rule at a time as the counts come down. The counts
# below are the baseline at the time ty was introduced (2026-09-07).
[tool.ty.rules]
# 415, of which ~279 are Django's `Model.objects` - ty has no django-stubs plugin support,
# so the default manager is invisible to it. Not fixable in this repo today.
unresolved-attribute = "warn"
# 138. Mostly Django/numpy/pandas dynamism plus genuinely loose annotations.
invalid-argument-type = "warn"
# 37.
invalid-assignment = "warn"
# 31. Optional ML extras (torch, transformers, sentence-transformers), the pybloomfilter
# native module, and scripts that import their siblings by bare module name.
unresolved-import = "warn"
# 19.
not-iterable = "warn"
# 12.
not-subscriptable = "warn"
# 10.
unsupported-operator = "warn"
# 9.
invalid-parameter-default = "warn"
# 8.
invalid-method-override = "warn"
# 6.
no-matching-overload = "warn"
# 4.
invalid-return-type = "warn"
# 2.
invalid-type-form = "warn"
# 0 - clean, and kept blocking. Calling something with the wrong number of arguments is
# always a bug, and ty catches it across modules where a test never would.
too-many-positional-arguments = "error"
missing-argument = "error"
# 1.
empty-body = "warn"
# 1.
call-non-callable = "warn"