Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added open telemetry instrumentation related items in backend service (#354)

Ritwik G450bd26d7699
diff --git a/backend/gunicorn.conf.py b/backend/gunicorn.conf.py
new file mode 100644
index 000000000..b158153df
--- /dev/null
+++ b/backend/gunicorn.conf.py
@@ -0,0 +1,35 @@
+import os
+from uuid import uuid4
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import SERVICE_INSTANCE_ID, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+# Retrieve the collector endpoint from the environment variable
+collector_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+
+
+def post_fork(server, worker):
+ server.log.info("Worker spawned (pid: %s)", worker.pid)
+ if not collector_endpoint:
+ server.log.error("OTEL_EXPORTER_OTLP_ENDPOINT environment variable is not set.")
+ return
+ try:
+ resource = Resource.create(
+ attributes={
+ # each worker needs a unique service.instance.id to
+ # distinguish the created metrics in prometheus
+ SERVICE_INSTANCE_ID: str(uuid4()),
+ "worker": worker.pid,
+ }
+ )
+
+ tracer_provider = TracerProvider(resource=resource)
+ tracer_provider.add_span_processor(
+ BatchSpanProcessor(OTLPSpanExporter(endpoint=collector_endpoint))
+ )
+ trace.set_tracer_provider(tracer_provider)
+ except Exception as e:
+ server.log.error("OTEL Error: %s", e)
diff --git a/docker/dockerfiles/backend.Dockerfile b/docker/dockerfiles/backend.Dockerfile
index a06f3ec00..19dc5e03c 100644
--- a/docker/dockerfiles/backend.Dockerfile
+++ b/docker/dockerfiles/backend.Dockerfile
@@ -13,6 +13,12 @@ ENV BUILD_PACKAGES_PATH unstract
ENV DJANGO_SETTINGS_MODULE "backend.settings.dev"
ENV PDM_VERSION 2.12.3
+# Disable all telemetry by default
+ENV OTEL_TRACES_EXPORTER none
+ENV OTEL_METRICS_EXPORTER none
+ENV OTEL_LOGS_EXPORTER none
+ENV OTEL_SERVICE_NAME unstract_backend
+
RUN apt-get update; \
apt-get --no-install-recommends install -y \
# unstract sdk
@@ -37,6 +43,12 @@ RUN set -e; \
# source command may not be availble in sh
. .venv/bin/activate; \
\
+ # Install opentelemetry for instrumentation.
+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \
+ \
+ opentelemetry-bootstrap -a install; \
+ \
+ # Applicaiton dependencies.
pdm sync --prod --no-editable; \
\
# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv