LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
refactor: Dockerfile optimization to build faster (#1126)
Chandrasekharan M44d63d8e02f6
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 6e80808bc..698790443 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -1,12 +1,16 @@---# See https://pre-commit.com for more information# See https://pre-commit.com/hooks.html for more hooks-# - Added unstract feature flag auto generated code to flake8 exclude list-# Force all unspecified python hooks to run python 3.10+default_language_version:python: python3.9default_stages:- pre-commit++ci:+ skip:+ - hadolint-docker # Fails in pre-commit CI+repos:- repo: https://github.com/pre-commit/pre-commit-hooksrev: v5.0.0@@ -99,17 +103,17 @@ repos:rev: v8.18.4hooks:- id: gitleaks- # - repo: https://github.com/hadolint/hadolint- # rev: v2.12.1-beta- # hooks:- # - id: hadolint-docker- # args:- # - --ignore=DL3003- # - --ignore=DL3008- # - --ignore=DL3013- # - --ignore=DL3018- # - --ignore=SC1091- # files: Dockerfile$+ - repo: https://github.com/hadolint/hadolint+ rev: v2.12.1-beta+ hooks:+ - id: hadolint-docker+ args:+ - --ignore=DL3003+ - --ignore=DL3008+ - --ignore=DL3013+ - --ignore=DL3018+ - --ignore=SC1091+ files: Dockerfile$- repo: https://github.com/asottile/yesqarev: v1.5.0hooks:diff --git a/docker/dockerfiles/backend.Dockerfile b/docker/dockerfiles/backend.Dockerfileindex bdcc3457f..643e28fb0 100644--- a/docker/dockerfiles/backend.Dockerfile+++ b/docker/dockerfiles/backend.Dockerfile@@ -2,23 +2,23 @@ FROM python:3.9-slimLABEL maintainer="Zipstack Inc."-# Keeps Python from generating .pyc files in the container-ENV PYTHONDONTWRITEBYTECODE 1-# Set to immediately flush stdout and stderr streams without first buffering-ENV PYTHONUNBUFFERED 1-ENV PYTHONPATH /unstract--ENV BUILD_CONTEXT_PATH backend-ENV BUILD_PACKAGES_PATH unstract-ENV DJANGO_SETTINGS_MODULE "backend.settings.dev"-ENV PDM_VERSION 2.16.1--# Disable all telemetry by default-ENV OTEL_TRACES_EXPORTER none-ENV OTEL_METRICS_EXPORTER none-ENV OTEL_LOGS_EXPORTER none-ENV OTEL_SERVICE_NAME unstract_backend-+ENV \+ # Keeps Python from generating .pyc files in the container+ PYTHONDONTWRITEBYTECODE=1 \+ # Set to immediately flush stdout and stderr streams without first buffering+ PYTHONUNBUFFERED=1 \+ PYTHONPATH=/unstract \+ BUILD_CONTEXT_PATH=backend \+ BUILD_PACKAGES_PATH=unstract \+ DJANGO_SETTINGS_MODULE="backend.settings.dev" \+ PDM_VERSION=2.16.1 \+ # Disable all telemetry by default+ OTEL_TRACES_EXPORTER=none \+ OTEL_TRACES_EXPORTER=none \+ OTEL_LOGS_EXPORTER=none \+ OTEL_SERVICE_NAME=unstract_backend++# Install system dependenciesRUN apt-get update; \apt-get --no-install-recommends install -y \# unstract sdk@@ -31,28 +31,23 @@ RUN apt-get update; \WORKDIR /app-COPY ${BUILD_CONTEXT_PATH}/ .+# Create venv and install gunicorn and other deps in it+RUN pdm venv create -w virtualenv --with-pip && \+ . .venv/bin/activate && \+ pip install --no-cache-dir \+ gunicorn \+ # Install opentelemetry for instrumentation+ opentelemetry-distro \+ opentelemetry-exporter-otlp && \+ opentelemetry-bootstrap -a install++COPY ${BUILD_CONTEXT_PATH}/ /app/# Copy local dependency packagesCOPY ${BUILD_PACKAGES_PATH}/ /unstract-RUN set -e; \- \- rm -rf .venv .pdm* .python* 2>/dev/null; \- \- pdm venv create -w virtualenv --with-pip; \- # source command may not be availble in sh- . .venv/bin/activate; \- \- # Install opentelemetry for instrumentation.- pip install opentelemetry-distro opentelemetry-exporter-otlp; \- \- opentelemetry-bootstrap -a install; \- \- # Applicaiton dependencies.- pdm sync --prod --no-editable; \- \- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv- pip install --no-cache-dir gunicorn;+# Install dependencies+RUN . .venv/bin/activate && \+ pdm sync --prod --no-editableEXPOSE 8000diff --git a/docker/dockerfiles/backend.Dockerfile.dockerignore b/docker/dockerfiles/backend.Dockerfile.dockerignoreindex f2d927913..28e33a313 100644--- a/docker/dockerfiles/backend.Dockerfile.dockerignore+++ b/docker/dockerfiles/backend.Dockerfile.dockerignore@@ -52,6 +52,7 @@.pre-commit-config.yaml**/teststest*.py+**/*.egg-info!backendfrontenddiff --git a/docker/dockerfiles/platform.Dockerfile b/docker/dockerfiles/platform.Dockerfileindex b348a3d69..71fec13b9 100644--- a/docker/dockerfiles/platform.Dockerfile+++ b/docker/dockerfiles/platform.Dockerfile@@ -2,16 +2,17 @@ FROM python:3.9-slimLABEL maintainer="Zipstack Inc."-# Keeps Python from generating .pyc files in the container-ENV PYTHONDONTWRITEBYTECODE 1-# Set to immediately flush stdout and stderr streams without first buffering-ENV PYTHONUNBUFFERED 1-ENV PYTHONPATH /unstract--ENV BUILD_CONTEXT_PATH platform-service-ENV BUILD_PACKAGES_PATH unstract-ENV PDM_VERSION 2.16.1-+ENV \+ # Keeps Python from generating .pyc files in the container+ PYTHONDONTWRITEBYTECODE=1 \+ # Set to immediately flush stdout and stderr streams without first buffering+ PYTHONUNBUFFERED=1 \+ PYTHONPATH=/unstract \+ BUILD_CONTEXT_PATH=platform-service \+ BUILD_PACKAGES_PATH=unstract \+ PDM_VERSION=2.16.1++# Install system dependenciesRUN apt-get update; \apt-get --no-install-recommends install -y \# unstract sdk@@ -29,27 +30,25 @@ USER unstractWORKDIR /app-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .+# Create venv and install gunicorn and other deps in it+RUN pdm venv create -w virtualenv --with-pip && \+ . .venv/bin/activate && \+ pip install --no-cache-dir \+ gunicorn \+ # Install opentelemetry for instrumentation+ opentelemetry-distro \+ opentelemetry-exporter-otlp && \+ opentelemetry-bootstrap -a install++# Read and execute access to non-root user to avoid security hotspot+# Write access to specific sub-directory need to be explicitly provided if required+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/# Copy local dependency packagesCOPY --chown=unstract ${BUILD_PACKAGES_PATH} /unstract-RUN set -e; \- \- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \- \- pdm venv create -w virtualenv --with-pip; \- # source command may not be availble in sh- . .venv/bin/activate; \- \- # Install opentelemetry for instrumentation.- pip install opentelemetry-distro opentelemetry-exporter-otlp; \- \- opentelemetry-bootstrap -a install; \- \- pdm sync --prod --no-editable; \- \- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv- pip install --no-cache-dir gunicorn;+# Install dependencies+RUN . .venv/bin/activate && \+ pdm sync --prod --no-editableEXPOSE 3001diff --git a/docker/dockerfiles/platform.Dockerfile.dockerignore b/docker/dockerfiles/platform.Dockerfile.dockerignoreindex 9c0fd8ed1..ef5d0a4f3 100644--- a/docker/dockerfiles/platform.Dockerfile.dockerignore+++ b/docker/dockerfiles/platform.Dockerfile.dockerignore@@ -52,6 +52,7 @@.pre-commit-config.yaml**/teststest*.py+**/*.egg-infobackendfrontenddiff --git a/docker/dockerfiles/prompt.Dockerfile b/docker/dockerfiles/prompt.Dockerfileindex 2138b67ac..f3a4c8835 100644--- a/docker/dockerfiles/prompt.Dockerfile+++ b/docker/dockerfiles/prompt.Dockerfile@@ -2,17 +2,18 @@ FROM python:3.9-slimLABEL maintainer="Zipstack Inc."-# Keeps Python from generating .pyc files in the container-ENV PYTHONDONTWRITEBYTECODE 1-# Set to immediately flush stdout and stderr streams without first buffering-ENV PYTHONUNBUFFERED 1-ENV PYTHONPATH /unstract--ENV BUILD_CONTEXT_PATH prompt-service-ENV BUILD_PACKAGES_PATH unstract-ENV TARGET_PLUGINS_PATH src/unstract/prompt_service/plugins-ENV PDM_VERSION 2.16.1+ENV \+ # Keeps Python from generating .pyc files in the container+ PYTHONDONTWRITEBYTECODE=1 \+ # Set to immediately flush stdout and stderr streams without first buffering+ PYTHONUNBUFFERED=1 \+ PYTHONPATH=/unstract \+ BUILD_CONTEXT_PATH=prompt-service \+ BUILD_PACKAGES_PATH=unstract \+ TARGET_PLUGINS_PATH=src/unstract/prompt_service/plugins \+ PDM_VERSION=2.16.1+# Install system dependenciesRUN apt-get update; \apt-get --no-install-recommends install -y \# unstract sdk@@ -31,58 +32,39 @@ USER unstractWORKDIR /app-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .-# Copy local dependency packages+# Create venv and install gunicorn and other deps in it+RUN pdm venv create -w virtualenv --with-pip && \+ . .venv/bin/activate && \+ pip install --no-cache-dir \+ gunicorn \+ gevent \+ # Install opentelemetry for instrumentation+ opentelemetry-distro \+ opentelemetry-exporter-otlp && \+ opentelemetry-bootstrap -a install++# Read and execute access to non-root user to avoid security hotspot+# Write access to specific sub-directory need to be explicitly provided if required+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/+# Copy local dependenciesCOPY --chown=unstract ${BUILD_PACKAGES_PATH}/core /unstract/coreCOPY --chown=unstract ${BUILD_PACKAGES_PATH}/flags /unstract/flags-RUN set -e; \- \- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \- \- pdm venv create -w virtualenv --with-pip; \- # source command may not be availble in sh- . .venv/bin/activate; \- \- # Install opentelemetry for instrumentation.- pip install opentelemetry-distro opentelemetry-exporter-otlp; \- \- opentelemetry-bootstrap -a install; \- \- pdm sync --prod --no-editable; \- \- #- # Install plugins- #- for dir in "${TARGET_PLUGINS_PATH}"/*/; \- do \- # Remove trailing "/". \- dirpath=${dir%*/}; \- \- # If no plugins, final part on split by "/" is equal to "*". \- if [ "${dirpath##*/}" = "*" ]; then \- continue; \- fi; \- \- cd "$dirpath"; \- echo "Installing plugin: ${dirpath##*/}..."; \- \- # PDM reuses active venv unless PDM_IGNORE_ACTIVE_VENV is set.- pdm sync --prod --no-editable; \- \- cd -; \- done; \- #- #- #- \- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv- pip install --no-cache-dir gunicorn gevent; \- \- # Storage for prompt studio uploads- mkdir prompt-studio-data;+# Install dependencies and plugins (if any)+RUN . .venv/bin/activate && \+ pdm sync --prod --no-editable && \+ for dir in "${TARGET_PLUGINS_PATH}"/*/; do \+ dirpath=${dir%*/}; \+ if [ "${dirpath##*/}" != "*" ]; then \+ cd "$dirpath" && \+ echo "Installing plugin: ${dirpath##*/}..." && \+ pdm sync --prod --no-editable && \+ cd -; \+ fi; \+ done && \+ mkdir prompt-studio-dataEXPOSE 3003-# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug-CMD [ "./entrypoint.sh" ]+# Default command+CMD ["./entrypoint.sh"]diff --git a/docker/dockerfiles/prompt.Dockerfile.dockerignore b/docker/dockerfiles/prompt.Dockerfile.dockerignoreindex a6bb9c3b5..30b4506ec 100644--- a/docker/dockerfiles/prompt.Dockerfile.dockerignore+++ b/docker/dockerfiles/prompt.Dockerfile.dockerignore@@ -52,6 +52,7 @@.pre-commit-config.yaml**/teststest*.py+**/*.egg-infobackendfrontenddiff --git a/docker/dockerfiles/runner.Dockerfile b/docker/dockerfiles/runner.Dockerfileindex 8fd253e38..a6df6209d 100644--- a/docker/dockerfiles/runner.Dockerfile+++ b/docker/dockerfiles/runner.Dockerfile@@ -2,15 +2,15 @@ FROM python:3.9-slimLABEL maintainer="Zipstack Inc."-# Keeps Python from generating .pyc files in the container-ENV PYTHONDONTWRITEBYTECODE 1-# Set to immediately flush stdout and stderr streams without first buffering-ENV PYTHONUNBUFFERED 1-ENV PYTHONPATH /unstract--ENV BUILD_CONTEXT_PATH runner-ENV BUILD_PACKAGES_PATH unstract-ENV PDM_VERSION 2.16.1+ENV \+ # Keeps Python from generating .pyc files in the container+ PYTHONDONTWRITEBYTECODE=1 \+ # Set to immediately flush stdout and stderr streams without first buffering+ PYTHONUNBUFFERED=1 \+ PYTHONPATH=/unstract \+ BUILD_CONTEXT_PATH=runner \+ BUILD_PACKAGES_PATH=unstract \+ PDM_VERSION=2.16.1RUN apt-get update \&& apt-get --no-install-recommends install -y docker \@@ -20,30 +20,33 @@ RUN apt-get update \WORKDIR /app-COPY ${BUILD_CONTEXT_PATH} .+# Create venv and install gunicorn and other deps in it+RUN pdm venv create -w virtualenv --with-pip && \+ . .venv/bin/activate && \+ pip install --no-cache-dir \+ gunicorn \+ gevent \+ # Install opentelemetry for instrumentation+ opentelemetry-distro \+ opentelemetry-exporter-otlp && \+ opentelemetry-bootstrap -a install++COPY ${BUILD_CONTEXT_PATH} /app/# Copy local dependency packagesCOPY ${BUILD_PACKAGES_PATH}/core /unstract/coreCOPY ${BUILD_PACKAGES_PATH}/flags /unstract/flags-RUN set -e; \- \- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \- \- pdm venv create -w virtualenv --with-pip; \+RUN \# source command may not be availble in sh. .venv/bin/activate; \\- # Install opentelemetry for instrumentation.- pip install opentelemetry-distro opentelemetry-exporter-otlp; \- \- opentelemetry-bootstrap -a install; \- \pdm sync --prod --no-editable; \\- [ -f cloud_requirements.txt ] && pip install -r cloud_requirements.txt || { echo "cloud_requirements.txt does not exist";}; \- \- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv- pip install --no-cache-dir gunicorn gevent;+ if [ -f cloud_requirements.txt ]; then \+ pip install --no-cache-dir -r cloud_requirements.txt; \+ else \+ echo "cloud_requirements.txt does not exist"; \+ fiEXPOSE 5002diff --git a/docker/dockerfiles/runner.Dockerfile.dockerignore b/docker/dockerfiles/runner.Dockerfile.dockerignoreindex d71471940..51dc2dd3a 100644--- a/docker/dockerfiles/runner.Dockerfile.dockerignore+++ b/docker/dockerfiles/runner.Dockerfile.dockerignore@@ -52,6 +52,7 @@.pre-commit-config.yaml**/teststest*.py+**/*.egg-infobackendfrontenddiff --git a/docker/dockerfiles/x2text.Dockerfile b/docker/dockerfiles/x2text.Dockerfileindex dc5f3eae6..661904a2c 100644--- a/docker/dockerfiles/x2text.Dockerfile+++ b/docker/dockerfiles/x2text.Dockerfile@@ -2,13 +2,13 @@ FROM python:3.9-slimLABEL maintainer="Zipstack Inc."-# Keeps Python from generating .pyc files in the container-ENV PYTHONDONTWRITEBYTECODE 1-# Set to immediately flush stdout and stderr streams without first buffering-ENV PYTHONUNBUFFERED 1--ENV BUILD_CONTEXT_PATH x2text-service-ENV PDM_VERSION 2.16.1+ENV \+ # Keeps Python from generating .pyc files in the container+ PYTHONDONTWRITEBYTECODE=1 \+ # Set to immediately flush stdout and stderr streams without first buffering+ PYTHONUNBUFFERED=1 \+ BUILD_CONTEXT_PATH=x2text-service \+ PDM_VERSION=2.16.1RUN pip install --no-cache-dir -U pip pdm~=${PDM_VERSION}; \\@@ -20,25 +20,23 @@ USER unstractWORKDIR /app-COPY --chown=unstract ${BUILD_CONTEXT_PATH} .--RUN set -e; \- \- rm -rf .venv .pdm* .python* requirements.txt 2>/dev/null; \- \- pdm venv create -w virtualenv --with-pip; \- # source command may not be availble in sh- . .venv/bin/activate; \- \- # Install opentelemetry for instrumentation.- pip install opentelemetry-distro opentelemetry-exporter-otlp; \- \- opentelemetry-bootstrap -a install; \- \- pdm sync --prod --no-editable; \- \- # REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv- pip install --no-cache-dir gunicorn;+# Create venv and install gunicorn and other deps in it+RUN pdm venv create -w virtualenv --with-pip && \+ . .venv/bin/activate && \+ pip install --no-cache-dir \+ gunicorn \+ # Install opentelemetry for instrumentation+ opentelemetry-distro \+ opentelemetry-exporter-otlp && \+ opentelemetry-bootstrap -a install++# Read and execute access to non-root user to avoid security hotspot+# Write access to specific sub-directory need to be explicitly provided if required+COPY --chmod=755 ${BUILD_CONTEXT_PATH} /app/++# Install dependencies+RUN . .venv/bin/activate && \+ pdm sync --prod --no-editableEXPOSE 3004diff --git a/docker/dockerfiles/x2text.Dockerfile.dockerignore b/docker/dockerfiles/x2text.Dockerfile.dockerignoreindex 5a00e5dd7..f40a80798 100644--- a/docker/dockerfiles/x2text.Dockerfile.dockerignore+++ b/docker/dockerfiles/x2text.Dockerfile.dockerignore@@ -52,6 +52,7 @@.pre-commit-config.yaml**/teststest*.py+**/*.egg-infobackendfrontend