An open source, non-profit web search engine
Migrate from poetry to uv
Daoud Clarke6e331f437b4a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4305aab..1068d25 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -35,15 +35,13 @@ jobs:with:python-version: '3.11.12'#----------------------------------------------- # ----- install & configure poetry -----+ # ----- install & configure uv -----#----------------------------------------------- - name: Install Poetry- uses: snok/[email protected]+ - name: Install uv+ uses: astral-sh/setup-uv@v5with:- version: 1.8.2- virtualenvs-create: true- virtualenvs-in-project: true- installer-parallel: true+ enable-cache: true+ cache-dependency-glob: "uv.lock"#----------------------------------------------# install Rust toolchain (required by maturin)@@ -52,26 +50,16 @@ jobs:uses: dtolnay/rust-toolchain@stable#----------------------------------------------- # load cached venv if cache exists- #----------------------------------------------- - name: Load cached venv- id: cached-poetry-dependencies- uses: actions/cache@v3- with:- path: .venv- key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}- #----------------------------------------------- # install dependencies if cache does not exist+ # install dependencies#----------------------------------------------- name: Install dependencies- if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'- run: poetry install --no-interaction --no-root+ run: uv sync --group dev#----------------------------------------------# build and install the project with maturin# (compiles the Rust extension and installs the package)#----------------------------------------------- name: Install project with maturin- run: poetry run maturin develop+ run: uv run maturin develop#----------------------------------------------# run test suite#----------------------------------------------@@ -83,4 +71,4 @@ jobs:DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_mwmblDJANGO_SETTINGS_MODULE: mwmbl.settings_testrun: |- poetry run pytest+ uv run pytestdiff --git a/Dockerfile b/Dockerfileindex d970ca5..ce51bfe 100644--- a/Dockerfile+++ b/Dockerfile@@ -15,9 +15,8 @@ WORKDIR /appFROM base as builder-ENV PIP_DEFAULT_TIMEOUT=100 \- PIP_DISABLE_PIP_VERSION_CHECK=1 \- PIP_NO_CACHE_DIR=1+# Install uv+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv# Install Rust toolchain (required by maturin to compile the mwmbl_rank extension)# Install clang/libclang-dev for bindgen (used by xgboost_lib-sys)@@ -32,20 +31,19 @@ ENV PATH="/root/.cargo/bin:${PATH}"# Create a /venv directory & environment.# This directory will be copied into the final stage of docker build.-RUN python -m venv /venv+RUN uv venv /venv# Copy only the necessary files to build/install the python package.# mwmbl/resources is needed because idf.rs and wiki.rs embed JSON files at# compile time via include_str!("../../mwmbl/resources/...").-COPY pyproject.toml poetry.lock /app/+COPY pyproject.toml uv.lock /app/COPY mwmbl /app/mwmblCOPY mwmbl_rank /app/mwmbl_rank# Working directory is /app-# Use pip to install the mwmbl python package including the mwmbl_rank Rust-# extension. PEP 517/518 allows pip to use maturin as the build backend.-RUN /venv/bin/pip install pip wheel --upgrade && \- /venv/bin/pip install .+# Use uv to install the mwmbl python package including the mwmbl_rank Rust+# extension. PEP 517/518 allows uv to use maturin as the build backend.+RUN uv pip install --python /venv/bin/python .# Copy libxgboost.so (prebuilt by the xgb crate) into the mwmbl_rank package# directory and patch the extension's RPATH so it finds the library at runtime.diff --git a/Dockerfile.crawler b/Dockerfile.crawlerindex ea33d8b..a0db7cc 100644--- a/Dockerfile.crawler+++ b/Dockerfile.crawler@@ -8,21 +8,20 @@ WORKDIR /appFROM base AS builder-# Install uv for fast Python package management-RUN pip install uv+# Install uv+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv# Create a /venv directory & environment.# This directory will be copied into the final stage of docker build.RUN uv venv /venv# Copy only the necessary files to build/install the python package-COPY pyproject.toml poetry.lock /app/+COPY pyproject.toml uv.lock /app/COPY mwmbl /app/mwmbl# Working directory is /app# Use uv to install the mwmbl python package-# uv is a fast Python package installer and resolver, compatible with pip-RUN uv pip install . --python /venv/bin/python+RUN uv pip install --python /venv/bin/python .FROM base AS finaldiff --git a/pyproject.toml b/pyproject.tomlindex 5f8ea81..a5bfb64 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,85 +1,12 @@-[tool.poetry]-name = "mwmbl"-version = "0.1.0"-description = ""-authors = ["Daoud Clarke <[email protected]>"]--[tool.poetry.dependencies]-python = ">=3.11,<3.12"-pandas = "^1.3.5"-scipy = "^1.8.0"-scikit-learn = "^1.0.2"-zstandard = "^0.16.0"-mmh3 = "^3.0.0"-uvicorn = "^0.16.0"-pyyaml = "==6.0"-boto3 = "1.35.99"-requests = "^2.27.1"-psycopg2-binary = "^2.9.3"-pytest = "^7.2.1"-pytest-mock = "^3.10.0"-pytest-django = "^4.5.2"--# Optional dependencies do not get installed by default. Look under tool.poetry.extras section-# to see which extras to use.-ujson = {version= "==4.3.0", optional = true}-warcio = {version= "==1.7.4", optional = true}-idna = {version= "==3.3", optional = true}-beautifulsoup4 = {version= "==4.10.0", optional = true}-langdetect = {version= "==1.0.9", optional = true}-pyarrow = {version= "==6.0.0", optional = true}-pyspark = {version= "==3.2.0", optional = true}-Levenshtein = {version= "==0.16.0", optional = true}-django = "^4.2.4"-django-ninja = "==1.3.0"-requests-cache = "^1.1.0"-redis = {extras = ["hiredis"], version = "^5.0.1"}-django-allauth = "==0.57.2"-dj-database-url = "^2.1.0"-django-htmx = "^1.17.0"-django-vite = "^2.1.3"-pybloomfiltermmap3 = "^0.6.0"-lxml-html-clean = "^0.1.0"-lxml = "^5.2.0"-numpy = "1.23.2"-objgraph = "^3.6.1"-psutil = "^6.0.0"-xgboost = "^2.1.0"-pydistinct = "^0.6.4"-django-ninja-jwt = "^5.3.3"-sentry-sdk = "^2.13.0"-django-debug-toolbar = "^5.2.0"-wasmtime = "^24.0.0"-dotenv = "^0.9.9"-py-spy = "^0.4.1"-scalar-ninja = "^0.3.0"--[tool.poetry.extras]-indexer = [- "ujson",- "warcio",- "idna",- "beautifulsoup4",- "langdetect",- "pyarrow",- "pyspark",- "Levenshtein",-]--[tool.poetry.group.dev.dependencies]-pytest = "^7.0.1"-fakeredis = "^2.30.1"-maturin = ">=1.5,<2.0"-[build-system]requires = ["maturin>=1.5,<2.0"]build-backend = "maturin"-# PEP 621 project metadata required by maturin to install entry point scripts.-# [tool.poetry.scripts] is Poetry-specific and ignored by maturin.+# PEP 621 project metadata[project]name = "mwmbl"version = "0.1.0"+description = ""requires-python = ">=3.11,<3.12"dependencies = ["pandas>=1.3.5,<2.0.0",@@ -117,10 +44,31 @@ dependencies = ["scalar-ninja>=0.3.0",]+[project.optional-dependencies]+indexer = [+ "ujson==4.3.0",+ "warcio==1.7.4",+ "idna==3.3",+ "beautifulsoup4==4.10.0",+ "langdetect==1.0.9",+ "pyarrow==6.0.0",+ "pyspark==3.2.0",+ "Levenshtein==0.16.0",+]+[project.scripts]mwmbl-tinysearchengine = "mwmbl.main:run"mwmbl-crawl = "mwmbl.crawl:run"+[dependency-groups]+dev = [+ "pytest>=7.0.1",+ "pytest-mock>=3.10.0",+ "pytest-django>=4.5.2",+ "fakeredis>=2.30.1",+ "maturin>=1.5,<2.0",+]+[tool.maturin]python-source = "."module-name = "mwmbl_rank"@@ -129,10 +77,6 @@ manifest-path = "mwmbl_rank/Cargo.toml"python-packages = ["mwmbl"]include = ["mwmbl_rank/data/hn_top_domains_filtered.json"]-[tool.poetry.scripts]-mwmbl-tinysearchengine = "mwmbl.main:run"-mwmbl-crawl = "mwmbl.crawl:run"-[tool.pytest.ini_options]log_cli = truelog_cli_level = "INFO"diff --git a/requirements.txt b/requirements.txtdeleted file mode 100644index 4a969c1..0000000--- a/requirements.txt+++ /dev/null@@ -1,16 +0,0 @@-zstandard~=0.16.0-pandas~=1.5.3-django~=4.2.6-boto3~=1.28.62-requests~=2.31.0-fastapi~=0.70.1-redis~=5.0.1-pydantic~=1.8.2-uvicorn~=0.16.0-urllib3~=2.0.6-justext~=3.0.0-scikit-learn~=1.3.1-mmh3~=3.1.0-numpy~=1.26.0-scipy~=1.11.3-pybloomfiltermmap3~=0.6.0