LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added open telemetry instrumentation related items in backend service (#354)
Ritwik G450bd26d7699
diff --git a/backend/gunicorn.conf.py b/backend/gunicorn.conf.pynew file mode 100644index 000000000..b158153df--- /dev/null+++ b/backend/gunicorn.conf.py@@ -0,0 +1,35 @@+import os+from uuid import uuid4++from opentelemetry import trace+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter+from opentelemetry.sdk.resources import SERVICE_INSTANCE_ID, Resource+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import BatchSpanProcessor++# Retrieve the collector endpoint from the environment variable+collector_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")+++def post_fork(server, worker):+ server.log.info("Worker spawned (pid: %s)", worker.pid)+ if not collector_endpoint:+ server.log.error("OTEL_EXPORTER_OTLP_ENDPOINT environment variable is not set.")+ return+ try:+ resource = Resource.create(+ attributes={+ # each worker needs a unique service.instance.id to+ # distinguish the created metrics in prometheus+ SERVICE_INSTANCE_ID: str(uuid4()),+ "worker": worker.pid,+ }+ )++ tracer_provider = TracerProvider(resource=resource)+ tracer_provider.add_span_processor(+ BatchSpanProcessor(OTLPSpanExporter(endpoint=collector_endpoint))+ )+ trace.set_tracer_provider(tracer_provider)+ except Exception as e:+ server.log.error("OTEL Error: %s", e)diff --git a/docker/dockerfiles/backend.Dockerfile b/docker/dockerfiles/backend.Dockerfileindex a06f3ec00..19dc5e03c 100644--- a/docker/dockerfiles/backend.Dockerfile+++ b/docker/dockerfiles/backend.Dockerfile@@ -13,6 +13,12 @@ ENV BUILD_PACKAGES_PATH unstractENV DJANGO_SETTINGS_MODULE "backend.settings.dev"ENV PDM_VERSION 2.12.3+# Disable all telemetry by default+ENV OTEL_TRACES_EXPORTER none+ENV OTEL_METRICS_EXPORTER none+ENV OTEL_LOGS_EXPORTER none+ENV OTEL_SERVICE_NAME unstract_backend+RUN apt-get update; \apt-get --no-install-recommends install -y \# unstract sdk@@ -37,6 +43,12 @@ RUN set -e; \# source command may not be availble in sh. .venv/bin/activate; \\+ # Install opentelemetry for instrumentation.+ pip install opentelemetry-distro opentelemetry-exporter-otlp; \+ \+ opentelemetry-bootstrap -a install; \+ \+ # Applicaiton dependencies.pdm sync --prod --no-editable; \\# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv