An open source, non-profit web search engine
Fix crawler docker
Daoud Clarke304eeeab6f5a
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.ymlindex e15b158..50b792e 100644--- a/.github/workflows/docker-publish.yml+++ b/.github/workflows/docker-publish.yml@@ -1,4 +1,4 @@-name: Build and Push Docker image+name: Build and Push Docker image for the crawleron:push:@@ -22,7 +22,7 @@ jobs:username: ${{ secrets.DOCKERHUB_USERNAME }}password: ${{ secrets.DOCKERHUB_TOKEN }}- - name: Build and push Docker image+ - name: Build and push Crawler Docker imageuses: docker/build-push-action@v5with:context: .diff --git a/Dockerfile.crawler b/Dockerfile.crawlerindex a0db7cc..a030753 100644--- a/Dockerfile.crawler+++ b/Dockerfile.crawler@@ -11,18 +11,37 @@ FROM base AS builder# Install uvCOPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv+# Install Rust toolchain (required by maturin to compile the mwmbl_rank extension)+# Install clang/libclang-dev for bindgen (used by xgboost_lib-sys)+# Install patchelf so maturin can bundle libxgboost.so into the wheel RPATH+RUN apt-get update && \+ apt-get install -y clang libclang-dev patchelf curl && \+ rm -rf /var/lib/apt/lists/*++RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \+ | sh -s -- -y --default-toolchain stable+ENV PATH="/root/.cargo/bin:${PATH}"+# Create a /venv directory & environment.# This directory will be copied into the final stage of docker build.RUN uv venv /venv-# Copy only the necessary files to build/install the python package+# Copy only the necessary files to build/install the python package.+# mwmbl_rank is needed because maturin compiles the Rust extension.COPY pyproject.toml uv.lock /app/COPY mwmbl /app/mwmbl+COPY mwmbl_rank /app/mwmbl_rank# Working directory is /app-# Use uv to install the mwmbl python package+# Use uv to install the mwmbl python package including the mwmbl_rank Rust extension.RUN uv pip install --python /venv/bin/python .+# Copy libxgboost.so (prebuilt by the xgb crate) into the mwmbl_rank package+# directory and patch the extension's RPATH so it finds the library at runtime.+RUN SODIR=/venv/lib/python3.11/site-packages/mwmbl_rank && \+ cp /app/mwmbl_rank/target/release/deps/libxgboost.so "$SODIR/" && \+ patchelf --set-rpath '$ORIGIN' "$SODIR/mwmbl_rank.cpython-311-x86_64-linux-gnu.so"+FROM base AS final# Copy only the required /venv directory from the builder image that contains mwmbl and its dependencies