Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

refactor: Dockerfile optimization to build faster (#1126)

Chandrasekharan M44d63d8e02f6
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6e80808bc..698790443 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,12 +1,16 @@
---
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
-# - Added unstract feature flag auto generated code to flake8 exclude list
-# Force all unspecified python hooks to run python 3.10
+
default_language_version:
python: python3.9
default_stages:
- pre-commit
+
+ci:
+ skip:
+ - hadolint-docker # Fails in pre-commit CI
+
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
@@ -99,17 +103,17 @@ repos:
rev: v8.18.4
hooks:
- id: gitleaks
- # - repo: https://github.com/hadolint/hadolint
- # rev: v2.12.1-beta
- # hooks:
- # - id: hadolint-docker
- # args:
- # - --ignore=DL3003
- # - --ignore=DL3008
- # - --ignore=DL3013
- # - --ignore=DL3018
- # - --ignore=SC1091
- # files: Dockerfile$
+ - repo: https://github.com/hadolint/hadolint
+ rev: v2.12.1-beta
+ hooks:
+ - id: hadolint-docker
+ args:
+ - --ignore=DL3003
+ - --ignore=DL3008
+ - --ignore=DL3013
+ - --ignore=DL3018
+ - --ignore=SC1091
+ files: Dockerfile$
- repo: https://github.com/asottile/yesqa
rev: v1.5.0
hooks:
diff --git a/docker/dockerfiles/backend.Dockerfile b/docker/dockerfiles/backend.Dockerfile
index bdcc3457f..643e28fb0 100644
--- a/docker/dockerfiles/backend.Dockerfile
+++ b/docker/dockerfiles/backend.Dockerfile
@@ -2,23 +2,23 @@ FROM python:3.9-slim
LABEL maintainer="Zipstack Inc."
-# Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE 1
-# Set to immediately flush stdout and stderr streams without first buffering
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /unstract
-
-ENV BUILD_CONTEXT_PATH backend
-ENV BUILD_PACKAGES_PATH unstract
-ENV DJANGO_SETTINGS_MODULE "backend.settings.dev"
-ENV PDM_VERSION 2.16.1
-
-# Disable all telemetry by default
-ENV OTEL_TRACES_EXPORTER none
-ENV OTEL_METRICS_EXPORTER none
-ENV OTEL_LOGS_EXPORTER none
-ENV OTEL_SERVICE_NAME unstract_backend
-
+ENV \
+ # Keeps Python from generating .pyc files in the container
+ PYTHONDONTWRITEBYTECODE=1 \
+ # Set to immediately flush stdout and stderr streams without first buffering
+ PYTHONUNBUFFERED=1 \
+ PYTHONPATH=/unstract \
+ BUILD_CONTEXT_PATH=backend \
+ BUILD_PACKAGES_PATH=unstract \
+ DJANGO_SETTINGS_MODULE="backend.settings.dev" \
+ PDM_VERSION=2.16.1 \
+ # Disable all telemetry by default
+ OTEL_TRACES_EXPORTER=none \
+ OTEL_TRACES_EXPORTER=none \
+ OTEL_LOGS_EXPORTER=none \
+ OTEL_SERVICE_NAME=unstract_backend
+
+# Install system dependencies
RUN apt-get update; \
apt-get --no-install-recommends install -y \
# unstract sdk
@@ -31,28 +31,23 @@ RUN apt-get update; \
WORKDIR /app
-COPY ${BUILD_CONTEXT_PATH}/ .
+# Create venv and install gunicorn and other deps in it
+RUN pdm venv create -w virtualenv --with-pip && \
+ . .venv/bin/activate && \
+ pip install --no-cache-dir \
+ gunicorn \
+ # Install opentelemetry for instrumentation
+ opentelemetry-distro \
+ opentelemetry-exporter-otlp && \
+ opentelemetry-bootstrap -a install
+
+COPY ${BUILD_CONTEXT_PATH}/ /app/
# Copy local dependency packages
COPY ${BUILD_PACKAGES_PATH}/ /unstract
-RUN set -e; \
- \
- rm -rf .venv .pdm* .python* 2>/dev/null; \
- \
- pdm venv create -w virtualenv --with-pip; \
- # source command may not be availble in sh
- . .venv/bin/activate; \
- \
- # Install opentelemetry for instrumentation.
- pip install opentelemetry-distro opentelemetry-exporter-otlp; \
- \
- opentelemetry-bootstrap -a install; \
- \
- # Applicaiton dependencies.
- pdm sync --prod --no-editable; \
- \
- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
- pip install --no-cache-dir gunicorn;
+# Install dependencies
+RUN . .venv/bin/activate && \
+ pdm sync --prod --no-editable
EXPOSE 8000
diff --git a/docker/dockerfiles/backend.Dockerfile.dockerignore b/docker/dockerfiles/backend.Dockerfile.dockerignore
index f2d927913..28e33a313 100644
--- a/docker/dockerfiles/backend.Dockerfile.dockerignore
+++ b/docker/dockerfiles/backend.Dockerfile.dockerignore
@@ -52,6 +52,7 @@
.pre-commit-config.yaml
**/tests
test*.py
+**/*.egg-info
!backend
frontend
diff --git a/docker/dockerfiles/platform.Dockerfile b/docker/dockerfiles/platform.Dockerfile
index b348a3d69..71fec13b9 100644
--- a/docker/dockerfiles/platform.Dockerfile
+++ b/docker/dockerfiles/platform.Dockerfile
@@ -2,16 +2,17 @@ FROM python:3.9-slim
LABEL maintainer="Zipstack Inc."
-# Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE 1
-# Set to immediately flush stdout and stderr streams without first buffering
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /unstract
-
-ENV BUILD_CONTEXT_PATH platform-service
-ENV BUILD_PACKAGES_PATH unstract
-ENV PDM_VERSION 2.16.1
-
+ENV \
+ # Keeps Python from generating .pyc files in the container
+ PYTHONDONTWRITEBYTECODE=1 \
+ # Set to immediately flush stdout and stderr streams without first buffering
+ PYTHONUNBUFFERED=1 \
+ PYTHONPATH=/unstract \
+ BUILD_CONTEXT_PATH=platform-service \
+ BUILD_PACKAGES_PATH=unstract \
+ PDM_VERSION=2.16.1
+
+# Install system dependencies
RUN apt-get update; \
apt-get --no-install-recommends install -y \
# unstract sdk
@@ -29,27 +30,25 @@ USER unstract
WORKDIR /app
-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .
+# Create venv and install gunicorn and other deps in it
+RUN pdm venv create -w virtualenv --with-pip && \
+ . .venv/bin/activate && \
+ pip install --no-cache-dir \
+ gunicorn \
+ # Install opentelemetry for instrumentation
+ opentelemetry-distro \
+ opentelemetry-exporter-otlp && \
+ opentelemetry-bootstrap -a install
+
+# Read and execute access to non-root user to avoid security hotspot
+# Write access to specific sub-directory need to be explicitly provided if required
+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
# Copy local dependency packages
COPY --chown=unstract ${BUILD_PACKAGES_PATH} /unstract
-RUN set -e; \
- \
- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \
- \
- pdm venv create -w virtualenv --with-pip; \
- # source command may not be availble in sh
- . .venv/bin/activate; \
- \
- # Install opentelemetry for instrumentation.
- pip install opentelemetry-distro opentelemetry-exporter-otlp; \
- \
- opentelemetry-bootstrap -a install; \
- \
- pdm sync --prod --no-editable; \
- \
- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
- pip install --no-cache-dir gunicorn;
+# Install dependencies
+RUN . .venv/bin/activate && \
+ pdm sync --prod --no-editable
EXPOSE 3001
diff --git a/docker/dockerfiles/platform.Dockerfile.dockerignore b/docker/dockerfiles/platform.Dockerfile.dockerignore
index 9c0fd8ed1..ef5d0a4f3 100644
--- a/docker/dockerfiles/platform.Dockerfile.dockerignore
+++ b/docker/dockerfiles/platform.Dockerfile.dockerignore
@@ -52,6 +52,7 @@
.pre-commit-config.yaml
**/tests
test*.py
+**/*.egg-info
backend
frontend
diff --git a/docker/dockerfiles/prompt.Dockerfile b/docker/dockerfiles/prompt.Dockerfile
index 2138b67ac..f3a4c8835 100644
--- a/docker/dockerfiles/prompt.Dockerfile
+++ b/docker/dockerfiles/prompt.Dockerfile
@@ -2,17 +2,18 @@ FROM python:3.9-slim
LABEL maintainer="Zipstack Inc."
-# Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE 1
-# Set to immediately flush stdout and stderr streams without first buffering
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /unstract
-
-ENV BUILD_CONTEXT_PATH prompt-service
-ENV BUILD_PACKAGES_PATH unstract
-ENV TARGET_PLUGINS_PATH src/unstract/prompt_service/plugins
-ENV PDM_VERSION 2.16.1
+ENV \
+ # Keeps Python from generating .pyc files in the container
+ PYTHONDONTWRITEBYTECODE=1 \
+ # Set to immediately flush stdout and stderr streams without first buffering
+ PYTHONUNBUFFERED=1 \
+ PYTHONPATH=/unstract \
+ BUILD_CONTEXT_PATH=prompt-service \
+ BUILD_PACKAGES_PATH=unstract \
+ TARGET_PLUGINS_PATH=src/unstract/prompt_service/plugins \
+ PDM_VERSION=2.16.1
+# Install system dependencies
RUN apt-get update; \
apt-get --no-install-recommends install -y \
# unstract sdk
@@ -31,58 +32,39 @@ USER unstract
WORKDIR /app
-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .
-# Copy local dependency packages
+# Create venv and install gunicorn and other deps in it
+RUN pdm venv create -w virtualenv --with-pip && \
+ . .venv/bin/activate && \
+ pip install --no-cache-dir \
+ gunicorn \
+ gevent \
+ # Install opentelemetry for instrumentation
+ opentelemetry-distro \
+ opentelemetry-exporter-otlp && \
+ opentelemetry-bootstrap -a install
+
+# Read and execute access to non-root user to avoid security hotspot
+# Write access to specific sub-directory need to be explicitly provided if required
+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
+# Copy local dependencies
COPY --chown=unstract ${BUILD_PACKAGES_PATH}/core /unstract/core
COPY --chown=unstract ${BUILD_PACKAGES_PATH}/flags /unstract/flags
-RUN set -e; \
- \
- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \
- \
- pdm venv create -w virtualenv --with-pip; \
- # source command may not be availble in sh
- . .venv/bin/activate; \
- \
- # Install opentelemetry for instrumentation.
- pip install opentelemetry-distro opentelemetry-exporter-otlp; \
- \
- opentelemetry-bootstrap -a install; \
- \
- pdm sync --prod --no-editable; \
- \
- #
- # Install plugins
- #
- for dir in "${TARGET_PLUGINS_PATH}"/*/; \
- do \
- # Remove trailing "/". \
- dirpath=${dir%*/}; \
- \
- # If no plugins, final part on split by "/" is equal to "*". \
- if [ "${dirpath##*/}" = "*" ]; then \
- continue; \
- fi; \
- \
- cd "$dirpath"; \
- echo "Installing plugin: ${dirpath##*/}..."; \
- \
- # PDM reuses active venv unless PDM_IGNORE_ACTIVE_VENV is set.
- pdm sync --prod --no-editable; \
- \
- cd -; \
- done; \
- #
- #
- #
- \
- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
- pip install --no-cache-dir gunicorn gevent; \
- \
- # Storage for prompt studio uploads
- mkdir prompt-studio-data;
+# Install dependencies and plugins (if any)
+RUN . .venv/bin/activate && \
+ pdm sync --prod --no-editable && \
+ for dir in "${TARGET_PLUGINS_PATH}"/*/; do \
+ dirpath=${dir%*/}; \
+ if [ "${dirpath##*/}" != "*" ]; then \
+ cd "$dirpath" && \
+ echo "Installing plugin: ${dirpath##*/}..." && \
+ pdm sync --prod --no-editable && \
+ cd -; \
+ fi; \
+ done && \
+ mkdir prompt-studio-data
EXPOSE 3003
-# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD [ "./entrypoint.sh" ]
+# Default command
+CMD ["./entrypoint.sh"]
diff --git a/docker/dockerfiles/prompt.Dockerfile.dockerignore b/docker/dockerfiles/prompt.Dockerfile.dockerignore
index a6bb9c3b5..30b4506ec 100644
--- a/docker/dockerfiles/prompt.Dockerfile.dockerignore
+++ b/docker/dockerfiles/prompt.Dockerfile.dockerignore
@@ -52,6 +52,7 @@
.pre-commit-config.yaml
**/tests
test*.py
+**/*.egg-info
backend
frontend
diff --git a/docker/dockerfiles/runner.Dockerfile b/docker/dockerfiles/runner.Dockerfile
index 8fd253e38..a6df6209d 100644
--- a/docker/dockerfiles/runner.Dockerfile
+++ b/docker/dockerfiles/runner.Dockerfile
@@ -2,15 +2,15 @@ FROM python:3.9-slim
LABEL maintainer="Zipstack Inc."
-# Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE 1
-# Set to immediately flush stdout and stderr streams without first buffering
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONPATH /unstract
-
-ENV BUILD_CONTEXT_PATH runner
-ENV BUILD_PACKAGES_PATH unstract
-ENV PDM_VERSION 2.16.1
+ENV \
+ # Keeps Python from generating .pyc files in the container
+ PYTHONDONTWRITEBYTECODE=1 \
+ # Set to immediately flush stdout and stderr streams without first buffering
+ PYTHONUNBUFFERED=1 \
+ PYTHONPATH=/unstract \
+ BUILD_CONTEXT_PATH=runner \
+ BUILD_PACKAGES_PATH=unstract \
+ PDM_VERSION=2.16.1
RUN apt-get update \
&& apt-get --no-install-recommends install -y docker \
@@ -20,30 +20,33 @@ RUN apt-get update \
WORKDIR /app
-COPY ${BUILD_CONTEXT_PATH} .
+# Create venv and install gunicorn and other deps in it
+RUN pdm venv create -w virtualenv --with-pip && \
+ . .venv/bin/activate && \
+ pip install --no-cache-dir \
+ gunicorn \
+ gevent \
+ # Install opentelemetry for instrumentation
+ opentelemetry-distro \
+ opentelemetry-exporter-otlp && \
+ opentelemetry-bootstrap -a install
+
+COPY ${BUILD_CONTEXT_PATH} /app/
# Copy local dependency packages
COPY ${BUILD_PACKAGES_PATH}/core /unstract/core
COPY ${BUILD_PACKAGES_PATH}/flags /unstract/flags
-RUN set -e; \
- \
- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \
- \
- pdm venv create -w virtualenv --with-pip; \
+RUN \
# source command may not be availble in sh
. .venv/bin/activate; \
\
- # Install opentelemetry for instrumentation.
- pip install opentelemetry-distro opentelemetry-exporter-otlp; \
- \
- opentelemetry-bootstrap -a install; \
- \
pdm sync --prod --no-editable; \
\
- [ -f cloud_requirements.txt ] && pip install -r cloud_requirements.txt || { echo "cloud_requirements.txt does not exist";}; \
- \
- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
- pip install --no-cache-dir gunicorn gevent;
+ if [ -f cloud_requirements.txt ]; then \
+ pip install --no-cache-dir -r cloud_requirements.txt; \
+ else \
+ echo "cloud_requirements.txt does not exist"; \
+ fi
EXPOSE 5002
diff --git a/docker/dockerfiles/runner.Dockerfile.dockerignore b/docker/dockerfiles/runner.Dockerfile.dockerignore
index d71471940..51dc2dd3a 100644
--- a/docker/dockerfiles/runner.Dockerfile.dockerignore
+++ b/docker/dockerfiles/runner.Dockerfile.dockerignore
@@ -52,6 +52,7 @@
.pre-commit-config.yaml
**/tests
test*.py
+**/*.egg-info
backend
frontend
diff --git a/docker/dockerfiles/x2text.Dockerfile b/docker/dockerfiles/x2text.Dockerfile
index dc5f3eae6..661904a2c 100644
--- a/docker/dockerfiles/x2text.Dockerfile
+++ b/docker/dockerfiles/x2text.Dockerfile
@@ -2,13 +2,13 @@ FROM python:3.9-slim
LABEL maintainer="Zipstack Inc."
-# Keeps Python from generating .pyc files in the container
-ENV PYTHONDONTWRITEBYTECODE 1
-# Set to immediately flush stdout and stderr streams without first buffering
-ENV PYTHONUNBUFFERED 1
-
-ENV BUILD_CONTEXT_PATH x2text-service
-ENV PDM_VERSION 2.16.1
+ENV \
+ # Keeps Python from generating .pyc files in the container
+ PYTHONDONTWRITEBYTECODE=1 \
+ # Set to immediately flush stdout and stderr streams without first buffering
+ PYTHONUNBUFFERED=1 \
+ BUILD_CONTEXT_PATH=x2text-service \
+ PDM_VERSION=2.16.1
RUN pip install --no-cache-dir -U pip pdm~=${PDM_VERSION}; \
\
@@ -20,25 +20,23 @@ USER unstract
WORKDIR /app
-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .
-
-RUN set -e; \
- \
- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \
- \
- pdm venv create -w virtualenv --with-pip; \
- # source command may not be availble in sh
- . .venv/bin/activate; \
- \
- # Install opentelemetry for instrumentation.
- pip install opentelemetry-distro opentelemetry-exporter-otlp; \
- \
- opentelemetry-bootstrap -a install; \
- \
- pdm sync --prod --no-editable; \
- \
- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
- pip install --no-cache-dir gunicorn;
+# Create venv and install gunicorn and other deps in it
+RUN pdm venv create -w virtualenv --with-pip && \
+ . .venv/bin/activate && \
+ pip install --no-cache-dir \
+ gunicorn \
+ # Install opentelemetry for instrumentation
+ opentelemetry-distro \
+ opentelemetry-exporter-otlp && \
+ opentelemetry-bootstrap -a install
+
+# Read and execute access to non-root user to avoid security hotspot
+# Write access to specific sub-directory need to be explicitly provided if required
+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/
+
+# Install dependencies
+RUN . .venv/bin/activate && \
+ pdm sync --prod --no-editable
EXPOSE 3004
diff --git a/docker/dockerfiles/x2text.Dockerfile.dockerignore b/docker/dockerfiles/x2text.Dockerfile.dockerignore
index 5a00e5dd7..f40a80798 100644
--- a/docker/dockerfiles/x2text.Dockerfile.dockerignore
+++ b/docker/dockerfiles/x2text.Dockerfile.dockerignore
@@ -52,6 +52,7 @@
.pre-commit-config.yaml
**/tests
test*.py
+**/*.egg-info
backend
frontend