Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2429 [MISC] Added OTEL filter for handling missing trace / span ID errors (#1363)

Chandrasekharan Mc172c521f20b
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 47aa0680c..5fbfa7145 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -9,6 +9,7 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
+import logging
import os
from pathlib import Path
from urllib.parse import urlparse
@@ -181,12 +182,20 @@ ALLOWED_HOSTS = ["*"]
CSRF_TRUSTED_ORIGINS = [WEB_APP_ORIGIN_URL, WEB_APP_ORIGIN_URL_WITH_WILD_CARD]
CORS_ALLOW_ALL_ORIGINS = False
-# Determine if OpenTelemetry trace context should be included in logs
-OTEL_TRACE_CONTEXT = (
- " trace_id:%(otelTraceID)s span_id:%(otelSpanID)s"
- if os.environ.get("OTEL_TRACES_EXPORTER", "none").lower() != "none"
- else ""
-)
+# Request ID middleware settings
+LOG_REQUEST_ID_HEADER = "X-Request-ID"
+REQUEST_ID_RESPONSE_HEADER = "X-Request-ID"
+GENERATE_REQUEST_ID_IF_NOT_IN_HEADER = True
+NO_REQUEST_ID = "-"
+
+
+class OTelFieldFilter(logging.Filter):
+ def filter(self, record):
+ for attr in ["otelTraceID", "otelSpanID"]:
+ if not hasattr(record, attr):
+ setattr(record, attr, "-")
+ return True
+
# Logging configuration
LOGGING = {
@@ -194,15 +203,15 @@ LOGGING = {
"disable_existing_loggers": False,
"filters": {
"request_id": {"()": "log_request_id.filters.RequestIDFilter"},
+ "otel_ids": {"()": "backend.settings.base.OTelFieldFilter"},
},
"formatters": {
"enriched": {
"format": (
"%(levelname)s : [%(asctime)s]"
"{module:%(module)s process:%(process)d "
- "thread:%(thread)d request_id:%(request_id)s"
- + OTEL_TRACE_CONTEXT
- + "} :- %(message)s"
+ "thread:%(thread)d request_id:%(request_id)s "
+ "trace_id:%(otelTraceID)s span_id:%(otelSpanID)s} :- %(message)s"
),
},
"verbose": {
@@ -218,7 +227,7 @@ LOGGING = {
"console": {
"level": DEFAULT_LOG_LEVEL, # Set the desired logging level here
"class": "logging.StreamHandler",
- "filters": ["request_id"],
+ "filters": ["request_id", "otel_ids"],
"formatter": "enriched",
},
},
@@ -228,6 +237,7 @@ LOGGING = {
# Set the desired logging level here as well
},
}
+
SHARED_APPS = (
# Multitenancy
# "django_tenants",
@@ -532,11 +542,6 @@ SOCIAL_AUTH_GOOGLE_OAUTH2_AUTH_EXTRA_ARGUMENTS = {
}
SOCIAL_AUTH_GOOGLE_OAUTH2_USE_UNIQUE_USER_ID = True
-# Request ID middleware settings
-LOG_REQUEST_ID_HEADER = "X-Request-ID"
-REQUEST_ID_RESPONSE_HEADER = "X-Request-ID"
-GENERATE_REQUEST_ID_IF_NOT_IN_HEADER = True
-NO_REQUEST_ID = "N/A"
# Always keep this line at the bottom of the file.
if missing_settings:
diff --git a/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py b/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py
deleted file mode 100644
index 649cff547..000000000
--- a/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import logging
-
-from flask import g
-
-
-class RequestIDFilter(logging.Filter):
- """Filter to inject request ID into log records."""
-
- def filter(self, record):
- record.request_id = getattr(g, "request_id", "N/A") if g else "N/A"
- return True
diff --git a/tool-sidecar/src/unstract/tool_sidecar/__init__.py b/tool-sidecar/src/unstract/tool_sidecar/__init__.py
index e69de29bb..d0db44675 100644
--- a/tool-sidecar/src/unstract/tool_sidecar/__init__.py
+++ b/tool-sidecar/src/unstract/tool_sidecar/__init__.py
@@ -0,0 +1,55 @@
+import logging.config
+import os
+
+from .constants import Env
+
+
+class OTelFieldFilter(logging.Filter):
+ def filter(self, record):
+ for attr in ["otelTraceID", "otelSpanID"]:
+ if not hasattr(record, attr):
+ setattr(record, attr, "-")
+ return True
+
+
+class RequestIDFilter(logging.Filter):
+ """A filter that adds file_execution_id from environment to log records."""
+
+ def filter(self, record):
+ record.request_id = os.getenv(Env.FILE_EXECUTION_ID, "-")
+ return True
+
+
+LOG_LEVEL = os.getenv(Env.LOG_LEVEL, "INFO")
+logging.config.dictConfig(
+ {
+ "version": 1,
+ "disable_existing_loggers": False,
+ "filters": {
+ "request_id": {"()": RequestIDFilter},
+ "otel_ids": {"()": OTelFieldFilter},
+ },
+ "formatters": {
+ "enriched": {
+ "format": (
+ "%(levelname)s : [%(asctime)s]"
+ "{module:%(module)s process:%(process)d "
+ "thread:%(thread)d request_id:%(request_id)s "
+ "trace_id:%(otelTraceID)s span_id:%(otelSpanID)s} :- %(message)s"
+ ),
+ }
+ },
+ "handlers": {
+ "console": {
+ "level": LOG_LEVEL,
+ "class": "logging.StreamHandler",
+ "filters": ["request_id", "otel_ids"],
+ "formatter": "enriched",
+ },
+ },
+ "root": {
+ "handlers": ["console"],
+ "level": LOG_LEVEL,
+ },
+ }
+)
diff --git a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
index a4b7716af..532d124be 100644
--- a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
+++ b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
@@ -23,10 +23,6 @@ from unstract.core.utilities import redact_sensitive_string
from .constants import Env, LogLevel, LogType
from .dto import LogLineDTO
-logging.basicConfig(
- level=getattr(logging, os.getenv(Env.LOG_LEVEL, "INFO")),
- format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-)
logger = logging.getLogger(__name__)
@@ -122,7 +118,7 @@ class LogProcessor:
log_dict = self.get_valid_log_message(line)
if not log_dict:
- logger.info(f"[Tool: {self.container_name}] {line}")
+ logger.info(f"{line}")
return LogLineDTO()
log_type = log_dict.get("type")
@@ -136,19 +132,19 @@ class LogProcessor:
log_process_status = LogLineDTO()
if log_type == LogType.LOG:
if log_level == LogLevel.ERROR:
- logger.error(f"[Tool: {self.container_name}] {log_dict.get('log')}")
+ logger.error(f"{log_dict.get('log')}")
log_process_status.error = log_dict.get("log")
self._update_tool_execution_status(
status=ToolExecutionStatus.FAILED, error=log_dict.get("log")
)
else:
- logger.info(f"[Tool: {self.container_name}] {log_dict.get('log')}")
+ logger.info(f"{log_dict.get('log')}")
elif log_type == LogType.RESULT:
- logger.info(f"[Tool {self.container_name}] Completed running")
+ logger.info(f"Tool '{self.container_name}' completed running")
self._update_tool_execution_status(status=ToolExecutionStatus.SUCCESS)
return LogLineDTO(with_result=True)
elif log_type == LogType.UPDATE:
- logger.info(f"[Tool: {self.container_name}] Pushing UI updates")
+ logger.info("Pushing UI updates")
log_dict["component"] = self.tool_instance_id
log_dict[LogFieldName.EXECUTION_ID] = self.execution_id
diff --git a/unstract/core/src/unstract/core/flask/logging.py b/unstract/core/src/unstract/core/flask/logging.py
index 950ad54d6..7056895a6 100644
--- a/unstract/core/src/unstract/core/flask/logging.py
+++ b/unstract/core/src/unstract/core/flask/logging.py
@@ -1,5 +1,4 @@
import logging
-import os
from logging.config import dictConfig
from flask import g
@@ -9,7 +8,15 @@ class RequestIDFilter(logging.Filter):
"""Filter to inject request ID into log records."""
def filter(self, record):
- record.request_id = getattr(g, "request_id", "N/A") if g else "N/A"
+ record.request_id = getattr(g, "request_id", "-") if g else "-"
+ return True
+
+
+class OTelFieldFilter(logging.Filter):
+ def filter(self, record):
+ for attr in ["otelTraceID", "otelSpanID"]:
+ if not hasattr(record, attr):
+ setattr(record, attr, "-")
return True
@@ -19,13 +26,6 @@ def setup_logging(log_level: int):
log_level (int): Log level to use. Can be one of
logging.INFO, logging.DEBUG, logging.WARNING, logging.ERROR
"""
- # Determine if OpenTelemetry trace context should be included in logs
- otel_trace_context = (
- " trace_id:%(otelTraceID)s span_id:%(otelSpanID)s"
- if os.environ.get("OTEL_TRACES_EXPORTER", "none").lower() != "none"
- else ""
- )
-
dictConfig(
{
"version": 1,
@@ -34,24 +34,26 @@ def setup_logging(log_level: int):
"default": {
"format": (
"%(levelname)s : [%(asctime)s]"
- "{pid:%(process)d tid:%(thread)d request_id:%(request_id)s}"
- + otel_trace_context
- + " "
- "%(name)s:- %(message)s"
+ "{pid:%(process)d tid:%(thread)d request_id:%(request_id)s "
+ + "trace_id:%(otelTraceID)s span_id:%(otelSpanID)s "
+ + "%(name)s}:- %(message)s"
),
},
},
"filters": {
"request_id": {
"()": RequestIDFilter,
- }
+ },
+ "otel_ids": {
+ "()": OTelFieldFilter,
+ },
},
"handlers": {
"wsgi": {
"class": "logging.StreamHandler",
"stream": "ext://flask.logging.wsgi_errors_stream",
"formatter": "default",
- "filters": ["request_id"],
+ "filters": ["request_id", "otel_ids"],
},
},
"loggers": {