Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fix crawler docker

Daoud Clarke304eeeab6f5a
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index e15b158..50b792e 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -1,4 +1,4 @@
-name: Build and Push Docker image
+name: Build and Push Docker image for the crawler
on:
push:
@@ -22,7 +22,7 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push Docker image
+ - name: Build and push Crawler Docker image
uses: docker/build-push-action@v5
with:
context: .
diff --git a/Dockerfile.crawler b/Dockerfile.crawler
index a0db7cc..a030753 100644
--- a/Dockerfile.crawler
+++ b/Dockerfile.crawler
@@ -11,18 +11,37 @@ FROM base AS builder
# Install uv
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+# Install Rust toolchain (required by maturin to compile the mwmbl_rank extension)
+# Install clang/libclang-dev for bindgen (used by xgboost_lib-sys)
+# Install patchelf so maturin can bundle libxgboost.so into the wheel RPATH
+RUN apt-get update && \
+ apt-get install -y clang libclang-dev patchelf curl && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+ | sh -s -- -y --default-toolchain stable
+ENV PATH="/root/.cargo/bin:${PATH}"
+
# Create a /venv directory & environment.
# This directory will be copied into the final stage of docker build.
RUN uv venv /venv
-# Copy only the necessary files to build/install the python package
+# Copy only the necessary files to build/install the python package.
+# mwmbl_rank is needed because maturin compiles the Rust extension.
COPY pyproject.toml uv.lock /app/
COPY mwmbl /app/mwmbl
+COPY mwmbl_rank /app/mwmbl_rank
# Working directory is /app
-# Use uv to install the mwmbl python package
+# Use uv to install the mwmbl python package including the mwmbl_rank Rust extension.
RUN uv pip install --python /venv/bin/python .
+# Copy libxgboost.so (prebuilt by the xgb crate) into the mwmbl_rank package
+# directory and patch the extension's RPATH so it finds the library at runtime.
+RUN SODIR=/venv/lib/python3.11/site-packages/mwmbl_rank && \
+ cp /app/mwmbl_rank/target/release/deps/libxgboost.so "$SODIR/" && \
+ patchelf --set-rpath '$ORIGIN' "$SODIR/mwmbl_rank.cpython-311-x86_64-linux-gnu.so"
+
FROM base AS final
# Copy only the required /venv directory from the builder image that contains mwmbl and its dependencies