Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Migrate from poetry to uv

Daoud Clarke6e331f437b4a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4305aab..1068d25 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,15 +35,13 @@ jobs:
with:
python-version: '3.11.12'
#----------------------------------------------
- # ----- install & configure poetry -----
+ # ----- install & configure uv -----
#----------------------------------------------
- - name: Install Poetry
- uses: snok/[email protected]
+ - name: Install uv
+ uses: astral-sh/setup-uv@v5
with:
- version: 1.8.2
- virtualenvs-create: true
- virtualenvs-in-project: true
- installer-parallel: true
+ enable-cache: true
+ cache-dependency-glob: "uv.lock"
#----------------------------------------------
# install Rust toolchain (required by maturin)
@@ -52,26 +50,16 @@ jobs:
uses: dtolnay/rust-toolchain@stable
#----------------------------------------------
- # load cached venv if cache exists
- #----------------------------------------------
- - name: Load cached venv
- id: cached-poetry-dependencies
- uses: actions/cache@v3
- with:
- path: .venv
- key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
- #----------------------------------------------
- # install dependencies if cache does not exist
+ # install dependencies
#----------------------------------------------
- name: Install dependencies
- if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
- run: poetry install --no-interaction --no-root
+ run: uv sync --group dev
#----------------------------------------------
# build and install the project with maturin
# (compiles the Rust extension and installs the package)
#----------------------------------------------
- name: Install project with maturin
- run: poetry run maturin develop
+ run: uv run maturin develop
#----------------------------------------------
# run test suite
#----------------------------------------------
@@ -83,4 +71,4 @@ jobs:
DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_mwmbl
DJANGO_SETTINGS_MODULE: mwmbl.settings_test
run: |
- poetry run pytest
+ uv run pytest
diff --git a/Dockerfile b/Dockerfile
index d970ca5..ce51bfe 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,9 +15,8 @@ WORKDIR /app
FROM base as builder
-ENV PIP_DEFAULT_TIMEOUT=100 \
- PIP_DISABLE_PIP_VERSION_CHECK=1 \
- PIP_NO_CACHE_DIR=1
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
# Install Rust toolchain (required by maturin to compile the mwmbl_rank extension)
# Install clang/libclang-dev for bindgen (used by xgboost_lib-sys)
@@ -32,20 +31,19 @@ ENV PATH="/root/.cargo/bin:${PATH}"
# Create a /venv directory & environment.
# This directory will be copied into the final stage of docker build.
-RUN python -m venv /venv
+RUN uv venv /venv
# Copy only the necessary files to build/install the python package.
# mwmbl/resources is needed because idf.rs and wiki.rs embed JSON files at
# compile time via include_str!("../../mwmbl/resources/...").
-COPY pyproject.toml poetry.lock /app/
+COPY pyproject.toml uv.lock /app/
COPY mwmbl /app/mwmbl
COPY mwmbl_rank /app/mwmbl_rank
# Working directory is /app
-# Use pip to install the mwmbl python package including the mwmbl_rank Rust
-# extension. PEP 517/518 allows pip to use maturin as the build backend.
-RUN /venv/bin/pip install pip wheel --upgrade && \
- /venv/bin/pip install .
+# Use uv to install the mwmbl python package including the mwmbl_rank Rust
+# extension. PEP 517/518 allows uv to use maturin as the build backend.
+RUN uv pip install --python /venv/bin/python .
# Copy libxgboost.so (prebuilt by the xgb crate) into the mwmbl_rank package
# directory and patch the extension's RPATH so it finds the library at runtime.
diff --git a/Dockerfile.crawler b/Dockerfile.crawler
index ea33d8b..a0db7cc 100644
--- a/Dockerfile.crawler
+++ b/Dockerfile.crawler
@@ -8,21 +8,20 @@ WORKDIR /app
FROM base AS builder
-# Install uv for fast Python package management
-RUN pip install uv
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
# Create a /venv directory & environment.
# This directory will be copied into the final stage of docker build.
RUN uv venv /venv
# Copy only the necessary files to build/install the python package
-COPY pyproject.toml poetry.lock /app/
+COPY pyproject.toml uv.lock /app/
COPY mwmbl /app/mwmbl
# Working directory is /app
# Use uv to install the mwmbl python package
-# uv is a fast Python package installer and resolver, compatible with pip
-RUN uv pip install . --python /venv/bin/python
+RUN uv pip install --python /venv/bin/python .
FROM base AS final
diff --git a/pyproject.toml b/pyproject.toml
index 5f8ea81..a5bfb64 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,85 +1,12 @@
-[tool.poetry]
-name = "mwmbl"
-version = "0.1.0"
-description = ""
-authors = ["Daoud Clarke <[email protected]>"]
-
-[tool.poetry.dependencies]
-python = ">=3.11,<3.12"
-pandas = "^1.3.5"
-scipy = "^1.8.0"
-scikit-learn = "^1.0.2"
-zstandard = "^0.16.0"
-mmh3 = "^3.0.0"
-uvicorn = "^0.16.0"
-pyyaml = "==6.0"
-boto3 = "1.35.99"
-requests = "^2.27.1"
-psycopg2-binary = "^2.9.3"
-pytest = "^7.2.1"
-pytest-mock = "^3.10.0"
-pytest-django = "^4.5.2"
-
-# Optional dependencies do not get installed by default. Look under tool.poetry.extras section
-# to see which extras to use.
-ujson = {version= "==4.3.0", optional = true}
-warcio = {version= "==1.7.4", optional = true}
-idna = {version= "==3.3", optional = true}
-beautifulsoup4 = {version= "==4.10.0", optional = true}
-langdetect = {version= "==1.0.9", optional = true}
-pyarrow = {version= "==6.0.0", optional = true}
-pyspark = {version= "==3.2.0", optional = true}
-Levenshtein = {version= "==0.16.0", optional = true}
-django = "^4.2.4"
-django-ninja = "==1.3.0"
-requests-cache = "^1.1.0"
-redis = {extras = ["hiredis"], version = "^5.0.1"}
-django-allauth = "==0.57.2"
-dj-database-url = "^2.1.0"
-django-htmx = "^1.17.0"
-django-vite = "^2.1.3"
-pybloomfiltermmap3 = "^0.6.0"
-lxml-html-clean = "^0.1.0"
-lxml = "^5.2.0"
-numpy = "1.23.2"
-objgraph = "^3.6.1"
-psutil = "^6.0.0"
-xgboost = "^2.1.0"
-pydistinct = "^0.6.4"
-django-ninja-jwt = "^5.3.3"
-sentry-sdk = "^2.13.0"
-django-debug-toolbar = "^5.2.0"
-wasmtime = "^24.0.0"
-dotenv = "^0.9.9"
-py-spy = "^0.4.1"
-scalar-ninja = "^0.3.0"
-
-[tool.poetry.extras]
-indexer = [
- "ujson",
- "warcio",
- "idna",
- "beautifulsoup4",
- "langdetect",
- "pyarrow",
- "pyspark",
- "Levenshtein",
-]
-
-[tool.poetry.group.dev.dependencies]
-pytest = "^7.0.1"
-fakeredis = "^2.30.1"
-maturin = ">=1.5,<2.0"
-
[build-system]
requires = ["maturin>=1.5,<2.0"]
build-backend = "maturin"
-# PEP 621 project metadata required by maturin to install entry point scripts.
-# [tool.poetry.scripts] is Poetry-specific and ignored by maturin.
+# PEP 621 project metadata
[project]
name = "mwmbl"
version = "0.1.0"
+description = ""
requires-python = ">=3.11,<3.12"
dependencies = [
"pandas>=1.3.5,<2.0.0",
@@ -117,10 +44,31 @@ dependencies = [
"scalar-ninja>=0.3.0",
]
+[project.optional-dependencies]
+indexer = [
+ "ujson==4.3.0",
+ "warcio==1.7.4",
+ "idna==3.3",
+ "beautifulsoup4==4.10.0",
+ "langdetect==1.0.9",
+ "pyarrow==6.0.0",
+ "pyspark==3.2.0",
+ "Levenshtein==0.16.0",
+]
+
[project.scripts]
mwmbl-tinysearchengine = "mwmbl.main:run"
mwmbl-crawl = "mwmbl.crawl:run"
+[dependency-groups]
+dev = [
+ "pytest>=7.0.1",
+ "pytest-mock>=3.10.0",
+ "pytest-django>=4.5.2",
+ "fakeredis>=2.30.1",
+ "maturin>=1.5,<2.0",
+]
+
[tool.maturin]
python-source = "."
module-name = "mwmbl_rank"
@@ -129,10 +77,6 @@ manifest-path = "mwmbl_rank/Cargo.toml"
python-packages = ["mwmbl"]
include = ["mwmbl_rank/data/hn_top_domains_filtered.json"]
-[tool.poetry.scripts]
-mwmbl-tinysearchengine = "mwmbl.main:run"
-mwmbl-crawl = "mwmbl.crawl:run"
-
[tool.pytest.ini_options]
log_cli = true
log_cli_level = "INFO"
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 4a969c1..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-zstandard~=0.16.0
-pandas~=1.5.3
-django~=4.2.6
-boto3~=1.28.62
-requests~=2.31.0
-fastapi~=0.70.1
-redis~=5.0.1
-pydantic~=1.8.2
-uvicorn~=0.16.0
-urllib3~=2.0.6
-justext~=3.0.0
-scikit-learn~=1.3.1
-mmh3~=3.1.0
-numpy~=1.26.0
-scipy~=1.11.3
-pybloomfiltermmap3~=0.6.0