Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

chore: add logs (#1743)

Edwin Jose1a059d2035d9
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 630024dd1..bf4472961 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -72,7 +72,7 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
- node-version: '20'
+ node-version: ${{ env.NODE_VERSION }}
- name: Install Python
run: uv python install 3.13
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 87a422196..f288b6521 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -8714,6 +8714,9 @@
"node": "^10 || ^12 || >=14"
}
},
+ "node_modules/next/node_modules/sharp": {
+ "optional": true
+ },
"node_modules/node-exports-info": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.0.tgz",
@@ -9246,7 +9249,6 @@
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
- "dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
diff --git a/sdks/typescript/src/documents.ts b/sdks/typescript/src/documents.ts
index d7f771aa1..274fce9a2 100644
--- a/sdks/typescript/src/documents.ts
+++ b/sdks/typescript/src/documents.ts
@@ -166,8 +166,8 @@ export class DocumentsClient {
}
const body: Record<string, string> = {};
- if (opts.filename) body.filename = opts.filename;
- if (opts.filterId) body.filter_id = opts.filterId;
+ if (opts.filename) body["filename"] = opts.filename;
+ if (opts.filterId) body["filter_id"] = opts.filterId;
try {
const response = await this.client._request("DELETE", "/api/v1/documents", {
diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts
index 871dc47e4..47a272bc9 100644
--- a/sdks/typescript/tests/integration.test.ts
+++ b/sdks/typescript/tests/integration.test.ts
@@ -234,7 +234,7 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
await client.documents.delete(alphaName);
await client.documents.delete(betaName);
}
- }, 60_000);
+ }, 120_000);
it("filterId in search actually scopes results to data_sources", async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-filter-"));
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.py
index 9007694e4..1e2368025 100644
--- a/src/api/settings/endpoints.py
+++ b/src/api/settings/endpoints.py
@@ -84,7 +84,7 @@ from services.docling_service import DoclingConfig, get_docling_preset_configs
from session_manager import User
from utils import provider_health_cache
from utils.langflow_utils import LangflowNotReadyError, wait_for_langflow
-from utils.logging_config import get_logger
+from utils.logging_config import get_logger, log_opensearch_env
from utils.telemetry import Category, MessageId, TelemetryClient
from utils.version_utils import OPENRAG_VERSION
@@ -779,6 +779,8 @@ async def onboarding(
try:
await TelemetryClient.send_event(Category.ONBOARDING, MessageId.ORB_ONBOARD_START)
+ log_opensearch_env(logger, "onboarding")
+
# Get current configuration
current_config = get_openrag_config()
diff --git a/src/app/lifespan.py b/src/app/lifespan.py
index c51b69693..ee481e589 100644
--- a/src/app/lifespan.py
+++ b/src/app/lifespan.py
@@ -25,7 +25,7 @@ from config.settings import (
get_opensearch_username,
)
from services.startup_orchestrator import startup_tasks
-from utils.logging_config import get_logger
+from utils.logging_config import get_logger, log_opensearch_env
from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
@@ -203,12 +203,15 @@ async def run_startup(app: FastAPI):
await mcp_lifespan_ctx.__aenter__()
logger.info("FastMCP lifespan started")
+ log_opensearch_env(logger, "startup")
+
# One-shot OpenSearch security bootstrap driven by the platform's
# service JWT. Runs synchronously (before startup_tasks) so the
# admin role mapping is in place before any other startup work
# talks to OpenSearch. The corresponding call inside startup_tasks
# is suppressed when this flag is on.
if OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP:
+ logger.info("OpenSearch security bootstrap enabled - starting")
from utils.opensearch_init import wait_for_opensearch
from utils.opensearch_utils import setup_opensearch_security
from utils.run_mode_utils import (
@@ -251,12 +254,15 @@ async def run_startup(app: FastAPI):
admin_username=admin_username,
)
try:
+ logger.info("Verifying OpenSearch readiness before bootstrap")
await wait_for_opensearch(opensearch_client)
logger.info("Bootstrapping OpenSearch security", admin_username=admin_username)
await setup_opensearch_security(opensearch_client, admin_username=admin_username)
logger.info("OpenSearch security bootstrap completed", admin_username=admin_username)
finally:
await opensearch_client.close()
+ else:
+ logger.info("OpenSearch security bootstrap disabled - skipping (handled in startup_tasks)")
# Start index initialization in background to avoid blocking OIDC endpoints
t1 = asyncio.create_task(startup_tasks(services))
diff --git a/src/auth/jwt_roles.py b/src/auth/jwt_roles.py
index 32d0134ba..50995ec1e 100644
--- a/src/auth/jwt_roles.py
+++ b/src/auth/jwt_roles.py
@@ -57,6 +57,11 @@ def extract_jwt_role_names(claims: dict | None) -> list[str]:
claim_name = get_jwt_roles_claim()
raw = claims.get(claim_name)
if raw is None:
+ logger.debug(
+ "JWT roles claim absent",
+ claim_name=claim_name,
+ available_claim_keys=list(claims.keys()),
+ )
return []
if not isinstance(raw, list) or not all(isinstance(v, str) for v in raw):
@@ -79,6 +84,12 @@ def extract_jwt_role_names(claims: dict | None) -> list[str]:
if role not in seen:
seen.add(role)
result.append(role)
+ logger.debug(
+ "JWT roles resolved",
+ claim_name=claim_name,
+ resolved_roles=result,
+ mapping_keys=list(mapping.keys()),
+ )
return result
diff --git a/src/config/utils.py b/src/config/utils.py
index d283cd0b0..813a5a9fc 100644
--- a/src/config/utils.py
+++ b/src/config/utils.py
@@ -100,10 +100,19 @@ def verify_jwt_from_issuer(
clients, pin the issuer instead.
"""
raw_token = _strip_bearer_prefix(token)
+ alg = kid = issuer = None
try:
header = jwt.get_unverified_header(raw_token)
- algorithm = header.get("alg")
+ alg = header.get("alg")
+ kid = header.get("kid")
+ algorithm = alg
if algorithm not in algorithms:
+ logger.debug(
+ "JWT rejected: unsupported alg",
+ alg=alg,
+ allowed=list(algorithms),
+ kid=kid,
+ )
return None
unverified_claims = jwt.decode(
@@ -112,6 +121,11 @@ def verify_jwt_from_issuer(
)
issuer = unverified_claims.get("iss")
if not isinstance(issuer, str) or not issuer:
+ logger.debug(
+ "JWT rejected: missing or invalid iss claim",
+ alg=alg,
+ kid=kid,
+ )
return None
public_key = get_public_key_from_issuer(
@@ -133,5 +147,13 @@ def verify_jwt_from_issuer(
decode_kwargs["audience"] = audience
return jwt.decode(raw_token, public_key, **decode_kwargs)
- except (ValueError, httpx.HTTPError, jwt.InvalidTokenError):
+ except (ValueError, httpx.HTTPError, jwt.InvalidTokenError) as e:
+ logger.debug(
+ "JWT verification failed",
+ error=str(e),
+ error_type=type(e).__name__,
+ alg=alg,
+ kid=kid,
+ iss=issuer,
+ )
return None
diff --git a/src/dependencies.py b/src/dependencies.py
index 36b77b123..97d8bb6bf 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -544,7 +544,13 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
# configured header; use it as the source of identity and roles. When RBAC is
# off, preserve the existing ibm-openrag-session cookie flow.
if jwt_roles_enabled():
- raw_jwt = request.headers.get(get_jwt_auth_header(), "")
+ header_name = get_jwt_auth_header()
+ raw_jwt = request.headers.get(header_name, "")
+ logger.debug(
+ "[AUTH] JWT-role header lookup",
+ header_name=header_name,
+ jwt_present=bool(raw_jwt and raw_jwt.strip()),
+ )
ibm_token = (
raw_jwt[7:].strip() if raw_jwt.startswith("Bearer ") else raw_jwt.strip()
) or None
@@ -802,6 +808,11 @@ async def get_api_key_user_async(
from config.utils import verify_jwt_from_issuer
raw_jwt = request.headers.get(get_jwt_auth_header(), "")
+ logger.debug(
+ "[AUTH] API-key path JWT header lookup",
+ header_name=get_jwt_auth_header(),
+ jwt_present=bool(raw_jwt and raw_jwt.strip()),
+ )
if raw_jwt and raw_jwt.strip():
token = raw_jwt[7:].strip() if raw_jwt.startswith("Bearer ") else raw_jwt.strip()
claims = verify_jwt_from_issuer(token, verify_tls=True)
diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py
index 2bd99dfe2..e4ac702e7 100644
--- a/src/utils/logging_config.py
+++ b/src/utils/logging_config.py
@@ -3,10 +3,9 @@ import logging.config
import os
import re
import sys
-from typing import Any, Dict
+from typing import Any
import structlog
-from structlog import processors
# ---------------------------------------------------------------------------
# Constants
@@ -16,13 +15,13 @@ LOC_WIDTH_SHORT = 30
LOC_WIDTH_LONG = 60
LEVEL_COLORS = {
- "DEBUG": "\033[36m", # Cyan
- "INFO": "\033[32m", # Green
- "WARNING": "\033[33m", # Yellow
- "ERROR": "\033[31m", # Red
+ "DEBUG": "\033[36m", # Cyan
+ "INFO": "\033[32m", # Green
+ "WARNING": "\033[33m", # Yellow
+ "ERROR": "\033[31m", # Red
"CRITICAL": "\033[1;31m", # Bold red
}
-DIM = "\033[38;5;244m" # Medium grey
+DIM = "\033[38;5;244m" # Medium grey
RESET = "\033[0m"
_SENSITIVE_HEADER_RE = re.compile(
@@ -38,13 +37,14 @@ _shared_processors: list = []
# Standalone processors (module-level so they can be reused in stdlib bridge)
# ---------------------------------------------------------------------------
-def drop_color_message_key(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+
+def drop_color_message_key(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:
"""Remove uvicorn's duplicate color_message field when bridging via stdlib."""
event_dict.pop("color_message", None)
return event_dict
-def filter_health_and_metrics(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+def filter_health_and_metrics(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:
"""Drop log events for high-frequency health/metrics endpoints."""
path = event_dict.get("path", "")
if path in ("/health", "/metrics", "/healthz", "/docs", "/openapi.json"):
@@ -52,7 +52,7 @@ def filter_health_and_metrics(_, __, event_dict: Dict[str, Any]) -> Dict[str, An
return event_dict
-def suppress_third_party_noise(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+def suppress_third_party_noise(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:
"""Drop WARNING/INFO/DEBUG log lines that originate from installed packages.
Third-party libraries (opensearch-py, httpx, boto3 …) log every HTTP
@@ -61,15 +61,12 @@ def suppress_third_party_noise(_, __, event_dict: Dict[str, Any]) -> Dict[str, A
"""
pathname = event_dict.get("pathname", "")
level = event_dict.get("level", "info")
- if (
- (".venv" in pathname or "site-packages" in pathname)
- and level not in ("error", "critical")
- ):
+ if (".venv" in pathname or "site-packages" in pathname) and level not in ("error", "critical"):
raise structlog.DropEvent()
return event_dict
-def clean_log_location(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+def clean_log_location(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:
"""Shorten pathname to a package-relative form for readability.
Strips the leading venv/site-packages prefix so logs show
@@ -79,17 +76,19 @@ def clean_log_location(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
marker = "site-packages/"
idx = pathname.find(marker)
if idx != -1:
- event_dict["pathname"] = pathname[idx + len(marker):]
+ event_dict["pathname"] = pathname[idx + len(marker) :]
return event_dict
def add_global_fields_factory(service: str, env: str, version: str):
"""Return a processor that stamps every event with service metadata."""
- def processor(_, __, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+
+ def processor(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:
event_dict.setdefault("service", service)
event_dict.setdefault("env", env)
event_dict.setdefault("version", version)
return event_dict
+
return processor
@@ -97,18 +96,17 @@ def add_global_fields_factory(service: str, env: str, version: str):
# Security helper
# ---------------------------------------------------------------------------
+
def sanitize_headers(headers: dict) -> dict:
"""Return a copy of *headers* with values of sensitive keys masked."""
- return {
- k: "***" if _SENSITIVE_HEADER_RE.search(k) else v
- for k, v in headers.items()
- }
+ return {k: "***" if _SENSITIVE_HEADER_RE.search(k) else v for k, v in headers.items()}
# ---------------------------------------------------------------------------
# Main configuration
# ---------------------------------------------------------------------------
+
def configure_logging(
log_level: str = "INFO",
json_logs: bool = False,
@@ -137,9 +135,7 @@ def configure_logging(
]
if include_timestamps:
- base_processors.append(
- structlog.processors.TimeStamper(fmt="iso", utc=True)
- )
+ base_processors.append(structlog.processors.TimeStamper(fmt="iso", utc=True))
base_processors.append(
structlog.processors.CallsiteParameterAdder(
@@ -163,9 +159,7 @@ def configure_logging(
else:
# Development: human-readable with exception info as text
use_colors = (
- "NO_COLOR" not in os.environ
- and hasattr(sys.stderr, "isatty")
- and sys.stderr.isatty()
+ "NO_COLOR" not in os.environ and hasattr(sys.stderr, "isatty") and sys.stderr.isatty()
)
def custom_formatter(logger, log_method, event_dict):
@@ -244,55 +238,51 @@ def configure_stdlib_logging(log_level: str = "INFO") -> None:
Uvicorn access logs are suppressed here because the ASGI middleware handles
request logging with richer context (request_id, duration_ms).
"""
- foreign_chain = list(_shared_processors) + [
- structlog.stdlib.ProcessorFormatter.remove_processors_meta,
- drop_color_message_key,
- structlog.processors.JSONRenderer(),
- ]
-
- logging.config.dictConfig({
- "version": 1,
- "disable_existing_loggers": False,
- "handlers": {
- "structlog": {
- "class": "logging.StreamHandler",
- "formatter": "structlog",
- "stream": "ext://sys.stderr",
- }
- },
- "formatters": {
- "structlog": {
- "()": structlog.stdlib.ProcessorFormatter,
- "processors": [
- structlog.stdlib.ProcessorFormatter.remove_processors_meta,
- drop_color_message_key,
- structlog.processors.JSONRenderer(),
- ],
- "foreign_pre_chain": list(_shared_processors),
- }
- },
- "root": {"handlers": ["structlog"], "level": log_level},
- "loggers": {
- # Third-party libs: pass through at ERROR+ only.
- # suppress_third_party_noise processor drops WARNING/INFO/DEBUG from
- # site-packages at the structlog layer, but setting the stdlib level
- # to ERROR here prevents them from even entering the pipeline.
- "httpcore": {"level": "ERROR", "propagate": True},
- "httpx": {"level": "ERROR", "propagate": True},
- "urllib3": {"level": "ERROR", "propagate": True},
- "boto3": {"level": "ERROR", "propagate": True},
- "botocore": {"level": "ERROR", "propagate": True},
- # opensearch-py logs every HTTP request (incl. 401 health checks) at WARNING.
- # ERROR-only keeps the pipeline clean; true connection failures still surface.
- "opensearch": {"level": "ERROR", "propagate": True},
- "opensearchpy": {"level": "ERROR", "propagate": True},
- "opensearchpy.trace": {"level": "CRITICAL", "propagate": False},
- "elastic_transport": {"level": "ERROR", "propagate": True},
- # uvicorn.access is replaced by RequestLoggingMiddleware
- "uvicorn.access": {"level": "CRITICAL", "propagate": False},
- "uvicorn.error": {"level": "WARNING", "propagate": True},
- },
- })
+ logging.config.dictConfig(
+ {
+ "version": 1,
+ "disable_existing_loggers": False,
+ "handlers": {
+ "structlog": {
+ "class": "logging.StreamHandler",
+ "formatter": "structlog",
+ "stream": "ext://sys.stderr",
+ }
+ },
+ "formatters": {
+ "structlog": {
+ "()": structlog.stdlib.ProcessorFormatter,
+ "processors": [
+ structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+ drop_color_message_key,
+ structlog.processors.JSONRenderer(),
+ ],
+ "foreign_pre_chain": list(_shared_processors),
+ }
+ },
+ "root": {"handlers": ["structlog"], "level": log_level},
+ "loggers": {
+ # Third-party libs: pass through at ERROR+ only.
+ # suppress_third_party_noise processor drops WARNING/INFO/DEBUG from
+ # site-packages at the structlog layer, but setting the stdlib level
+ # to ERROR here prevents them from even entering the pipeline.
+ "httpcore": {"level": "ERROR", "propagate": True},
+ "httpx": {"level": "ERROR", "propagate": True},
+ "urllib3": {"level": "ERROR", "propagate": True},
+ "boto3": {"level": "ERROR", "propagate": True},
+ "botocore": {"level": "ERROR", "propagate": True},
+ # opensearch-py logs every HTTP request (incl. 401 health checks) at WARNING.
+ # ERROR-only keeps the pipeline clean; true connection failures still surface.
+ "opensearch": {"level": "ERROR", "propagate": True},
+ "opensearchpy": {"level": "ERROR", "propagate": True},
+ "opensearchpy.trace": {"level": "CRITICAL", "propagate": False},
+ "elastic_transport": {"level": "ERROR", "propagate": True},
+ # uvicorn.access is replaced by RequestLoggingMiddleware
+ "uvicorn.access": {"level": "CRITICAL", "propagate": False},
+ "uvicorn.error": {"level": "WARNING", "propagate": True},
+ },
+ }
+ )
def get_logger(name: str = None) -> structlog.BoundLogger:
@@ -302,6 +292,26 @@ def get_logger(name: str = None) -> structlog.BoundLogger:
return structlog.get_logger()
+def log_opensearch_env(logger, stage: str) -> None:
+ """Log effective OpenSearch-related env values for a given stage.
+
+ Logs the parsed booleans as resolved in config.settings for a given stage.
+ """
+ from config.settings import (
+ OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,
+ OPENRAG_SKIP_OS_SECURITY_SETUP,
+ )
+ from utils.run_mode_utils import get_run_mode
+
+ logger.info(
+ "OpenRAG run mode details",
+ stage=stage,
+ run_mode=get_run_mode(),
+ bootstrap_os_security_on_startup=OPENRAG_BOOTSTRAP_OS_SECURITY_ON_STARTUP,
+ skip_os_security_setup=OPENRAG_SKIP_OS_SECURITY_SETUP,
+ )
+
+
def configure_from_env() -> None:
"""Configure logging from environment variables."""
from utils.version_utils import OPENRAG_VERSION # avoid circular import at module level
diff --git a/src/utils/version_utils.py b/src/utils/version_utils.py
index 09c22a13a..060ab3b75 100644
--- a/src/utils/version_utils.py
+++ b/src/utils/version_utils.py
@@ -2,11 +2,12 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+
def _get_openrag_version() -> str:
"""Get OpenRAG version from package metadata."""
try:
- from importlib.metadata import version, PackageNotFoundError
-
+ from importlib.metadata import PackageNotFoundError, version
+
try:
for dist_name in ["openrag", "openrag-nightly"]:
try:
@@ -19,22 +20,23 @@ def _get_openrag_version() -> str:
try:
import tomllib
from pathlib import Path
-
+
# Try to find pyproject.toml relative to this file
current_file = Path(__file__)
project_root = current_file.parent.parent.parent.parent
pyproject_path = project_root / "pyproject.toml"
-
+
if pyproject_path.exists():
with open(pyproject_path, "rb") as f:
data = tomllib.load(f)
return data.get("project", {}).get("version", "dev")
except Exception:
pass
-
+
return "dev"
except Exception as e:
logger.warning(f"Failed to get OpenRAG version: {e}")
return "unknown"
-OPENRAG_VERSION = _get_openrag_version()
\ No newline at end of file
+
+OPENRAG_VERSION = _get_openrag_version()