Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added open telemetry zero code instrumentation package (#802)

Ritwik G1f736cfbe904
diff --git a/backend/gunicorn_conf.py b/backend/gunicorn_conf.py
deleted file mode 100644
index b158153df..000000000
--- a/backend/gunicorn_conf.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import os
-from uuid import uuid4
-
-from opentelemetry import trace
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_INSTANCE_ID, Resource
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-
-# Retrieve the collector endpoint from the environment variable
-collector_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
-
-
-def post_fork(server, worker):
- server.log.info("Worker spawned (pid: %s)", worker.pid)
- if not collector_endpoint:
- server.log.error("OTEL_EXPORTER_OTLP_ENDPOINT environment variable is not set.")
- return
- try:
- resource = Resource.create(
- attributes={
- # each worker needs a unique service.instance.id to
- # distinguish the created metrics in prometheus
- SERVICE_INSTANCE_ID: str(uuid4()),
- "worker": worker.pid,
- }
- )
-
- tracer_provider = TracerProvider(resource=resource)
- tracer_provider.add_span_processor(
- BatchSpanProcessor(OTLPSpanExporter(endpoint=collector_endpoint))
- )
- trace.set_tracer_provider(tracer_provider)
- except Exception as e:
- server.log.error("OTEL Error: %s", e)
diff --git a/docker/dockerfiles/platform.Dockerfile b/docker/dockerfiles/platform.Dockerfile
index 2ba3aba4a..c659c261f 100644
--- a/docker/dockerfiles/platform.Dockerfile
+++ b/docker/dockerfiles/platform.Dockerfile
@@ -34,6 +34,11 @@ RUN set -e; \
# source command may not be availble in sh
. .venv/bin/activate; \
\
+ # Install opentelemetry for instrumentation.
+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \
+ \
+ opentelemetry-bootstrap -a install; \
+ \
pdm sync --prod --no-editable; \
\
# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
diff --git a/docker/dockerfiles/prompt.Dockerfile b/docker/dockerfiles/prompt.Dockerfile
index 1e0730135..9b80a4590 100644
--- a/docker/dockerfiles/prompt.Dockerfile
+++ b/docker/dockerfiles/prompt.Dockerfile
@@ -44,6 +44,11 @@ RUN set -e; \
# source command may not be availble in sh
. .venv/bin/activate; \
\
+ # Install opentelemetry for instrumentation.
+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \
+ \
+ opentelemetry-bootstrap -a install; \
+ \
pdm sync --prod --no-editable; \
\
#
diff --git a/docker/dockerfiles/worker.Dockerfile b/docker/dockerfiles/worker.Dockerfile
index 8a0337d38..8e49fddb0 100644
--- a/docker/dockerfiles/worker.Dockerfile
+++ b/docker/dockerfiles/worker.Dockerfile
@@ -32,6 +32,11 @@ RUN set -e; \
# source command may not be availble in sh
. .venv/bin/activate; \
\
+ # Install opentelemetry for instrumentation.
+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \
+ \
+ opentelemetry-bootstrap -a install; \
+ \
pdm sync --prod --no-editable; \
\
[ -f cloud_requirements.txt ] && pip install -r cloud_requirements.txt || { echo "cloud_requirements.txt does not exist";}; \
diff --git a/docker/dockerfiles/x2text.Dockerfile b/docker/dockerfiles/x2text.Dockerfile
index e2fc2d777..dc5f3eae6 100644
--- a/docker/dockerfiles/x2text.Dockerfile
+++ b/docker/dockerfiles/x2text.Dockerfile
@@ -30,6 +30,11 @@ RUN set -e; \
# source command may not be availble in sh
. .venv/bin/activate; \
\
+ # Install opentelemetry for instrumentation.
+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \
+ \
+ opentelemetry-bootstrap -a install; \
+ \
pdm sync --prod --no-editable; \
\
# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
diff --git a/tools/structure/Dockerfile b/tools/structure/Dockerfile
index 344ed6157..447540ccd 100644
--- a/tools/structure/Dockerfile
+++ b/tools/structure/Dockerfile
@@ -17,6 +17,7 @@ RUN pip install --no-cache-dir -U pip
WORKDIR /app
COPY requirements.txt /app/
RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install pip install opentelemetry-distro opentelemetry-exporter-otlp && opentelemetry-bootstrap -a install
# Copy the contents of your project directory into the container at /app
COPY src /app/src/
WORKDIR /app/src