An open source, non-profit web search engine
# Mwmbl development Makefile
#
# Prerequisites:
# - uv (https://github.com/astral-sh/uv)
# - PostgreSQL
# - Redis running on the configured REDIS_URL (default: 127.0.0.1:6379)
#
# Quick start:
# Set DATABASE_URL to a PostgreSQL database you can use for development.
# The PostgreSQL user must have CREATEDB so Django can create the test database.
#
# If PostgreSQL is running in Docker but Mwmbl is running directly on the host,
# use 127.0.0.1 (or localhost) and the published port. The Docker container name
# is only resolvable from containers on the same Docker network.
#
# make install # one-time setup for dev/test DBs
# make migrate # apply all migrations
# make test # run the full test suite
# make run # start the dev server
# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------
# Override these via environment or make invocation, e.g.:
# DATABASE_URL="postgres://user:pass@host/db" make test
# These have no defaults — you must set them in your environment or on the command line.
DATABASE_URL ?=
REDIS_URL ?= redis://127.0.0.1:6379
# A Redis database the tests flush, so keep it apart from REDIS_URL's.
TEST_REDIS_URL ?= redis://127.0.0.1:6379/15
# Python tests that cross into the mwmbl_rank extension, run against the instrumented build.
RUST_COVERAGE_TESTS ?= test/test_rust_features.py test/test_rust_pipeline.py test/test_indexer.py \
test/test_index_page_api.py test/test_index_page_writes.py test/test_copy_index.py
CARGO_HOME ?= $(HOME)/.cargo
DJANGO_SETTINGS_MODULE ?= mwmbl.settings_dev
# ---------------------------------------------------------------------------
# Targets
# ---------------------------------------------------------------------------
SODIR := .venv/lib/python3.11/site-packages/mwmbl_rank
XGB_SO := mwmbl_rank/target/release/deps/libxgboost.so
.PHONY: help install install-hooks patch-xgboost migrate test test-file coverage coverage-python coverage-rust \
coverage-quick run run-background \
check fix format format-check lint lint-fix typecheck typecheck-all locked
help:
@echo "Available targets:"
@echo " install Install all dependencies with uv"
@echo " migrate Apply Django migrations"
@echo " test Run the full test suite (set DATABASE_URL to a PostgreSQL database)"
@echo " test-file FILE Run a specific test file, e.g. make test-file FILE=test/test_search_api_key.py"
@echo " coverage Run Python and Rust coverage and print a combined line-weighted total"
@echo " coverage-python Run Python tests with terminal, XML, and HTML coverage reports"
@echo " coverage-rust Run Rust unit tests and write an LCOV report (needs cargo-llvm-cov)"
@echo " coverage-quick Run tests with coverage report (terminal only, faster)"
@echo " run Start the Django development server"
@echo " run-background Start the background task processor"
@echo ""
@echo " check Run every check CI runs: locked, format-check, lint, typecheck"
@echo " fix Auto-fix what can be auto-fixed: format + lint --fix"
@echo " format Reformat the code with ruff"
@echo " format-check Check formatting without writing (CI gate)"
@echo " lint Run ruff lint checks"
@echo " lint-fix Run ruff lint checks, applying safe fixes"
@echo " typecheck Run ty, failing only on error-level diagnostics (CI gate)"
@echo " typecheck-all Run ty showing the full warning backlog, never fails"
@echo " locked Check uv.lock is up to date with pyproject.toml (CI gate)"
@echo " install-hooks Install the pre-commit hooks into .git/hooks"
install:
uv sync
$(MAKE) patch-xgboost
# Copy libxgboost.so into the mwmbl_rank package dir and set RPATH to $ORIGIN,
# matching the Dockerfile post-build step so the extension is self-contained.
patch-xgboost:
@if [ -f "$(XGB_SO)" ]; then \
cp "$(XGB_SO)" "$(SODIR)/"; \
.venv/bin/patchelf --set-rpath '$$ORIGIN' "$(SODIR)/mwmbl_rank.cpython-311-x86_64-linux-gnu.so"; \
echo "Patched RPATH and copied libxgboost.so to $(SODIR)"; \
else \
echo "$(XGB_SO) not found — run 'uv run maturin develop' first"; \
fi
migrate:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" \
uv run python manage.py migrate --settings=$(DJANGO_SETTINGS_MODULE)
test:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" \
uv run pytest $(PYTEST_ARGS)
test-file:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" \
uv run pytest $(FILE) -v $(PYTEST_ARGS)
coverage: coverage-python coverage-rust
uv run python scripts/combine_coverage.py coverage/python.xml coverage/rust.lcov
coverage-python:
@mkdir -p coverage
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" \
uv run pytest --cov=mwmbl --cov-report=term-missing --cov-report=xml:coverage/python.xml \
--cov-report=html:coverage/python-html --ignore=mwmbl_rank/vendor $(PYTEST_ARGS)
coverage-rust:
@PATH="$(CARGO_HOME)/bin:$(PATH)" cargo llvm-cov --version >/dev/null 2>&1 || { \
echo "cargo-llvm-cov is required: rustup component add llvm-tools-preview && cargo install cargo-llvm-cov --locked"; \
exit 1; \
}
@mkdir -p coverage
@set -eu; \
PATH="$(CARGO_HOME)/bin:$(PATH)"; \
export PATH PYO3_PYTHON="$(CURDIR)/.venv/bin/python"; \
eval "$$(cd mwmbl_rank && cargo llvm-cov show-env --sh)"; \
cargo llvm-cov clean --manifest-path mwmbl_rank/Cargo.toml --workspace; \
cargo test --manifest-path mwmbl_rank/Cargo.toml --lib; \
uv run --no-sync maturin develop; \
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" \
uv run --no-sync pytest $(RUST_COVERAGE_TESTS); \
cargo llvm-cov report \
--manifest-path mwmbl_rank/Cargo.toml \
--lcov \
--output-path "$(CURDIR)/coverage/rust.lcov" \
--ignore-filename-regex '(/vendor/|/.cargo/registry/)'
coverage-quick:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" \
uv run pytest --cov=mwmbl --cov-report=term-missing --ignore=mwmbl_rank/vendor $(PYTEST_ARGS)
run:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" \
uv run python manage.py runserver --settings=$(DJANGO_SETTINGS_MODULE)
run-background:
DATABASE_URL="$(DATABASE_URL)" REDIS_URL="$(REDIS_URL)" \
uv run python manage.py process_tasks --settings=$(DJANGO_SETTINGS_MODULE)
# ---------------------------------------------------------------------------
# Checks
#
# `make check` is exactly what CI and the pre-commit hook run. Keep the three
# targets below in sync with .pre-commit-config.yaml and .github/workflows/ci.yml.
# ---------------------------------------------------------------------------
# `locked` comes first on purpose: every other target below shells out to `uv run`,
# which relocks uv.lock in place when pyproject.toml has moved on, and a lockfile that
# has just been rewritten always passes `uv lock --check`.
check: locked format-check lint typecheck
fix: format lint-fix
format:
uv run ruff format
format-check:
uv run ruff format --check
lint:
uv run ruff check
lint-fix:
uv run ruff check --fix
# Most rules are downgraded to "warn" in pyproject.toml (see [tool.ty.rules]) because
# they have a large pre-existing baseline - notably Django's `Model.objects`, which ty
# cannot see. Print the error-level diagnostics only, and fail on those alone, so the
# warning backlog does not bury a real regression.
typecheck:
@output=$$(uv run ty check --exit-zero-on-warning --output-format concise 2>&1); \
status=$$?; \
echo "$$output" | grep -E ' error\[' || true; \
echo "$$output" | tail -n 1; \
exit $$status
# The full backlog, warnings included. Advisory: never fails the build.
typecheck-all:
uv run ty check --exit-zero-on-warning
# The Docker images install with `uv sync --frozen`, which takes uv.lock exactly as
# committed, so a pyproject.toml edited without relocking would leave them building the
# old dependency set while CI and the tests run the new one.
locked:
uv lock --check
install-hooks:
uv run pre-commit install