LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added open telemetry zero code instrumentation package (#802)
Ritwik G1f736cfbe904
diff --git a/backend/gunicorn_conf.py b/backend/gunicorn_conf.pydeleted file mode 100644index b158153df..000000000--- a/backend/gunicorn_conf.py+++ /dev/null@@ -1,35 +0,0 @@-import os-from uuid import uuid4--from opentelemetry import trace-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter-from opentelemetry.sdk.resources import SERVICE_INSTANCE_ID, Resource-from opentelemetry.sdk.trace import TracerProvider-from opentelemetry.sdk.trace.export import BatchSpanProcessor--# Retrieve the collector endpoint from the environment variable-collector_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")---def post_fork(server, worker):- server.log.info("Worker spawned (pid: %s)", worker.pid)- if not collector_endpoint:- server.log.error("OTEL_EXPORTER_OTLP_ENDPOINT environment variable is not set.")- return- try:- resource = Resource.create(- attributes={- # each worker needs a unique service.instance.id to- # distinguish the created metrics in prometheus- SERVICE_INSTANCE_ID: str(uuid4()),- "worker": worker.pid,- }- )-- tracer_provider = TracerProvider(resource=resource)- tracer_provider.add_span_processor(- BatchSpanProcessor(OTLPSpanExporter(endpoint=collector_endpoint))- )- trace.set_tracer_provider(tracer_provider)- except Exception as e:- server.log.error("OTEL Error: %s", e)diff --git a/docker/dockerfiles/platform.Dockerfile b/docker/dockerfiles/platform.Dockerfileindex 2ba3aba4a..c659c261f 100644--- a/docker/dockerfiles/platform.Dockerfile+++ b/docker/dockerfiles/platform.Dockerfile@@ -34,6 +34,11 @@ RUN set -e; \# source command may not be availble in sh. .venv/bin/activate; \\+ # Install opentelemetry for instrumentation.+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \+ \+ opentelemetry-bootstrap -a install; \+ \pdm sync --prod --no-editable; \\# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenvdiff --git a/docker/dockerfiles/prompt.Dockerfile b/docker/dockerfiles/prompt.Dockerfileindex 1e0730135..9b80a4590 100644--- a/docker/dockerfiles/prompt.Dockerfile+++ b/docker/dockerfiles/prompt.Dockerfile@@ -44,6 +44,11 @@ RUN set -e; \# source command may not be availble in sh. .venv/bin/activate; \\+ # Install opentelemetry for instrumentation.+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \+ \+ opentelemetry-bootstrap -a install; \+ \pdm sync --prod --no-editable; \\#diff --git a/docker/dockerfiles/worker.Dockerfile b/docker/dockerfiles/worker.Dockerfileindex 8a0337d38..8e49fddb0 100644--- a/docker/dockerfiles/worker.Dockerfile+++ b/docker/dockerfiles/worker.Dockerfile@@ -32,6 +32,11 @@ RUN set -e; \# source command may not be availble in sh. .venv/bin/activate; \\+ # Install opentelemetry for instrumentation.+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \+ \+ opentelemetry-bootstrap -a install; \+ \pdm sync --prod --no-editable; \\[ -f cloud_requirements.txt ] && pip install -r cloud_requirements.txt || { echo "cloud_requirements.txt does not exist";}; \diff --git a/docker/dockerfiles/x2text.Dockerfile b/docker/dockerfiles/x2text.Dockerfileindex e2fc2d777..dc5f3eae6 100644--- a/docker/dockerfiles/x2text.Dockerfile+++ b/docker/dockerfiles/x2text.Dockerfile@@ -30,6 +30,11 @@ RUN set -e; \# source command may not be availble in sh. .venv/bin/activate; \\+ # Install opentelemetry for instrumentation.+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \+ \+ opentelemetry-bootstrap -a install; \+ \pdm sync --prod --no-editable; \\# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenvdiff --git a/tools/structure/Dockerfile b/tools/structure/Dockerfileindex 344ed6157..447540ccd 100644--- a/tools/structure/Dockerfile+++ b/tools/structure/Dockerfile@@ -17,6 +17,7 @@ RUN pip install --no-cache-dir -U pipWORKDIR /appCOPY requirements.txt /app/RUN pip install --no-cache-dir -r requirements.txt+RUN pip install pip install opentelemetry-distro opentelemetry-exporter-otlp && opentelemetry-bootstrap -a install# Copy the contents of your project directory into the container at /appCOPY src /app/src/WORKDIR /app/src