OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
refactor: main into app modules and route registrars (#1561)
Edwin Josebfc237830d61
diff --git a/.github/workflows/lint-backend.yml b/.github/workflows/lint-backend.ymlnew file mode 100644index 000000000..9233ccd2b--- /dev/null+++ b/.github/workflows/lint-backend.yml@@ -0,0 +1,85 @@+name: Lint Backend++on:+ pull_request:+ paths:+ - 'src/**/*.py'+ - 'tests/**/*.py'+ - 'pyproject.toml'+ - 'uv.lock'+ - '.github/workflows/lint-backend.yml'++# Cancel in-flight runs on the same PR when a new push lands.+concurrency:+ group: lint-backend-${{ github.ref }}+ cancel-in-progress: true++jobs:+ lint:+ name: Ruff and mypy on changed files+ runs-on: ubuntu-latest++ steps:+ - name: Checkout+ uses: actions/checkout@v4+ with:+ # Need the full base ref locally so we can compute the diff+ # against the PR's merge base.+ fetch-depth: 0++ - name: Install uv+ uses: astral-sh/setup-uv@v3+ with:+ enable-cache: true++ - name: Set up Python+ run: uv python install 3.13++ - name: Install dev dependencies+ run: uv sync --group dev++ - name: Compute changed Python files+ id: changed+ run: |+ base="${{ github.event.pull_request.base.sha }}"+ head="${{ github.event.pull_request.head.sha }}"+ # Diff against the merge base so we only see files touched in the PR.+ merge_base=$(git merge-base "$base" "$head")+ mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$merge_base" "$head" -- '*.py')+ if [ "${#files[@]}" -eq 0 ]; then+ echo "No Python files changed."+ echo "files=" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ printf '%s\n' "${files[@]}"+ # Pass as a single space-separated string to subsequent steps.+ echo "files=${files[*]}" >> "$GITHUB_OUTPUT"++ - name: Ruff (auto-fix dry run + format check)+ if: steps.changed.outputs.files != ''+ run: |+ # `--no-fix` reports the same set of issues that `--fix` would+ # apply, so the developer sees what to run locally to clean up.+ uv run ruff check --no-fix --output-format=github ${{ steps.changed.outputs.files }}+ uv run ruff format --check ${{ steps.changed.outputs.files }}++ - name: Mypy+ if: steps.changed.outputs.files != ''+ run: |+ # Run mypy from src/ so module paths resolve, dropping the+ # leading "src/" from each path. Tests files are checked from+ # the repo root.+ src_files=()+ test_files=()+ for f in ${{ steps.changed.outputs.files }}; do+ case "$f" in+ src/*) src_files+=("${f#src/}") ;;+ tests/*) test_files+=("$f") ;;+ esac+ done+ if [ "${#src_files[@]}" -gt 0 ]; then+ (cd src && uv run mypy "${src_files[@]}")+ fi+ if [ "${#test_files[@]}" -gt 0 ]; then+ uv run mypy "${test_files[@]}"+ fidiff --git a/pyproject.toml b/pyproject.tomlindex 8dcd999be..1a473ad2a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -63,7 +63,14 @@ dependencies = [][dependency-groups]-dev = ["pytest>=9.0.3", "pytest-asyncio>=0.21.0", "pytest-mock>=3.12.0", "pytest-cov>=4.0.0"]+dev = [+ "pytest>=9.0.3",+ "pytest-asyncio>=0.21.0",+ "pytest-mock>=3.12.0",+ "pytest-cov>=4.0.0",+ "ruff>=0.15.12",+ "mypy>=2.0.0",+][project.scripts]openrag = "tui.main:run_tui"@@ -77,4 +84,40 @@ filterwarnings = [package = trueoverride-dependencies = ["python-dotenv>=1.1.0"]+# Ruff and mypy run only against files changed in a PR (see+# .github/workflows/lint-backend.yml). The configs below stay+# intentionally minimal — broad enforcement is out of scope for the+# refactor that introduced them.+[tool.ruff]+line-length = 100+target-version = "py313"++[tool.ruff.lint]+select = ["E", "F", "I", "B", "UP"]+ignore = ["E501"] # line-length handled by formatter++# `bootstrap` must be the first import in any module that uses it+# (it loads .env and configures structlog before anything else can+# read env vars or emit logs). Define it as its own isort section+# placed above standard-library so the formatter never re-sorts it+# below other imports.+[tool.ruff.lint.isort]+section-order = [+ "future",+ "bootstrap",+ "standard-library",+ "third-party",+ "first-party",+ "local-folder",+]++[tool.ruff.lint.isort.sections]+bootstrap = ["bootstrap"]++[tool.mypy]+python_version = "3.13"+ignore_missing_imports = true+no_strict_optional = true+warn_unused_ignores = true+diff --git a/src/api/health.py b/src/api/health.pynew file mode 100644index 000000000..42278447f--- /dev/null+++ b/src/api/health.py@@ -0,0 +1,80 @@+"""Liveness and readiness probes."""++import asyncio++import httpx+from fastapi import Request+from fastapi.responses import JSONResponse++from config.settings import clients+from utils.logging_config import get_logger++logger = get_logger(__name__)+++async def health_check(request: Request):+ """Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""+ return JSONResponse({"status": "ok"}, status_code=200)+++async def opensearch_health_ready(request):+ """Readiness probe: verifies OpenSearch dependency is reachable."""+ from config.settings import IBM_AUTH_ENABLED, OPENSEARCH_URL++ if IBM_AUTH_ENABLED:+ logger.debug("[OPENSEARCH] OpenSearch auth mode enabled, health check per-request")+ # In IBM auth mode we cannot rely on the global OpenSearch client+ # (auth is established per-request), so perform a lightweight,+ # unauthenticated connectivity check against the OpenSearch endpoint.+ opensearch_url = OPENSEARCH_URL.rstrip("/")+ try:+ timeout = httpx.Timeout(5.0)+ async with httpx.AsyncClient(timeout=timeout) as client:+ resp = await client.get(f"{opensearch_url}/")+ if resp.status_code < 500:+ logger.debug("[OPENSEARCH] OpenSearch health check successful")+ return JSONResponse(+ {+ "status": "ready",+ "dependencies": {"opensearch": "up"},+ "note": "OpenSearch auth mode - connectivity verified via unauthenticated probe",+ },+ status_code=200,+ )+ else:+ logger.debug("[OPENSEARCH] OpenSearch health check failed")+ return JSONResponse(+ {+ "status": "not_ready",+ "dependencies": {"opensearch": "down"},+ "error": f"Unexpected status from OpenSearch: {resp.status_code}",+ },+ status_code=503,+ )+ except Exception as e:+ logger.error("[OPENSEARCH] OpenSearch health check failed", error=str(e))+ return JSONResponse(+ {+ "status": "not_ready",+ "dependencies": {"opensearch": "down"},+ "error": "OpenSearch health check failed",+ },+ status_code=503,+ )++ try:+ await asyncio.wait_for(clients.opensearch.info(), timeout=5.0)+ return JSONResponse(+ {"status": "ready", "dependencies": {"opensearch": "up"}},+ status_code=200,+ )+ except Exception as e:+ logger.error("[OPENSEARCH] OpenSearch health check failed", error=str(e))+ return JSONResponse(+ {+ "status": "not_ready",+ "dependencies": {"opensearch": "down"},+ "error": "OpenSearch health check failed",+ },+ status_code=503,+ )diff --git a/src/app/__init__.py b/src/app/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/src/app/container.py b/src/app/container.pynew file mode 100644index 000000000..8cbe394cc--- /dev/null+++ b/src/app/container.py@@ -0,0 +1,202 @@+"""Service container — constructs all application services and wires their dependencies.++Returns a dict consumed by routes (via FastAPI Depends) and the lifespan hook.+"""++from api.connector_router import ConnectorRouter+from config.settings import (+ INGESTION_TIMEOUT,+ JWT_SIGNING_KEY,+ SESSION_SECRET,+ clients,+ config_manager,+ get_embedding_model,+ get_index_name,+ is_no_auth_mode,+)+from connectors.langflow_connector_service import LangflowConnectorService+from connectors.service import ConnectorService+from services.api_key_service import APIKeyService+from services.auth_service import AuthService+from services.chat_service import ChatService+from services.document_service import DocumentService+from services.flows_service import FlowsService+from services.knowledge_filter_service import KnowledgeFilterService+from services.langflow_file_service import LangflowFileService+from services.langflow_mcp_service import LangflowMCPService+from services.models_service import ModelsService+from services.monitor_service import MonitorService+from services.search_service import SearchService, register_search_service+from services.task_service import TaskService+from session_manager import SessionManager+from utils.jwt_keygen import generate_jwt_keys+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClient++logger = get_logger(__name__)+++async def initialize_services():+ """Initialize all services and their dependencies"""+ await TelemetryClient.send_event(Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_START)+ from config.settings import IBM_AUTH_ENABLED++ if IBM_AUTH_ENABLED:+ logger.info("IBM auth mode enabled — JWT validation delegated to Traefik")++ # Generate JWT keys if they don't exist, a JWT signing key isn't specified,+ # and IBM auth is not enabled (IBM mode delegates all auth to Traefik)+ if not JWT_SIGNING_KEY and not IBM_AUTH_ENABLED:+ generate_jwt_keys()++ # Initialize clients (now async to generate Langflow API key)+ try:+ await clients.initialize()+ except Exception as e:+ logger.error("Failed to initialize clients", error=str(e))+ await TelemetryClient.send_event(+ Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_OS_CLIENT_FAIL+ )+ raise++ session_manager = SessionManager(SESSION_SECRET)++ models_service = ModelsService()+ document_service = DocumentService(+ session_manager=session_manager,+ models_service=models_service,+ docling_service=clients.docling_service,+ )+ search_service = SearchService(session_manager, models_service)+ register_search_service(search_service)+ task_service = TaskService(+ document_service,+ models_service,+ ingestion_timeout=INGESTION_TIMEOUT,+ docling_service=clients.docling_service,+ )+ flows_service = FlowsService()+ chat_service = ChatService(flows_service=flows_service)+ knowledge_filter_service = KnowledgeFilterService(session_manager)+ monitor_service = MonitorService(session_manager)+ langflow_file_service = LangflowFileService(+ flows_service=flows_service,+ docling_service=clients.docling_service,+ )+ langflow_mcp_service = LangflowMCPService()++ langflow_connector_service = LangflowConnectorService(+ task_service=task_service,+ session_manager=session_manager,+ flows_service=flows_service,+ docling_service=clients.docling_service,+ )+ openrag_connector_service = ConnectorService(+ patched_async_client=clients,+ embed_model=get_embedding_model(),+ index_name=get_index_name(),+ task_service=task_service,+ session_manager=session_manager,+ models_service=models_service,+ document_service=document_service,+ docling_service=clients.docling_service,+ )++ connector_service = ConnectorRouter(+ langflow_connector_service=langflow_connector_service,+ openrag_connector_service=openrag_connector_service,+ )++ auth_service = AuthService(+ session_manager,+ connector_service,+ flows_service,+ langflow_mcp_service=langflow_mcp_service,+ )++ # Load persisted connector connections at startup so webhooks and syncs+ # can resolve existing subscriptions immediately after server boot+ # Skip in no-auth mode since connectors require OAuth+ if not is_no_auth_mode():+ try:+ await connector_service.initialize()+ loaded_count = len(connector_service.connection_manager.connections)+ logger.info(+ "Loaded persisted connector connections on startup",+ loaded_count=loaded_count,+ )+ except Exception as e:+ logger.error("Failed to load persisted connections on startup", error=str(e))+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_LOAD_FAILED+ )+ else:+ logger.info("[CONNECTORS] Skipping connection loading in no-auth mode")++ await TelemetryClient.send_event(+ Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_SUCCESS+ )++ api_key_service = APIKeyService(session_manager)++ # ===== RBAC service =====+ # We do NOT open the SQL engine here. `create_app()` runs inside an+ # `asyncio.run(...)` whose loop is closed BEFORE uvicorn starts its+ # own loop — any AsyncEngine bound to this loop would be dead by+ # then. Instead, RBACService takes a *lazy* session-factory getter+ # that resolves `db.engine.SessionLocal` at call time — that+ # attribute is filled in by the lifespan startup event running on+ # uvicorn's loop. Alembic upgrade is run synchronously from __main__+ # before `asyncio.run(create_app())` so the schema is in place by+ # the time the lifespan opens the engine.+ from db import engine as _db_engine_mod+ from services.rbac_service import RBACService+ from services.workspace_config_service import WorkspaceConfigService++ def _lazy_session_factory():+ sl = _db_engine_mod.SessionLocal+ if sl is None:+ raise RuntimeError(+ "DB engine not yet initialized. RBACService called before lifespan startup."+ )+ return sl()++ rbac_service = RBACService(_lazy_session_factory)++ # WorkspaceConfigService — DB-first reads of what config.yaml holds,+ # with the legacy ConfigManager kept as the yaml fallback during+ # Phase B (dual-write).+ workspace_config_service = WorkspaceConfigService(+ config_manager=config_manager,+ session_factory=_lazy_session_factory,+ )++ # Plumb the session factory into the two chat-history services+ # (session_ownership + conversation_persistence). They lazy-resolve+ # `db.engine.SessionLocal` as a fallback, but setting it here makes+ # the wiring explicit and avoids the import path on the hot loop.+ from services.conversation_persistence_service import conversation_persistence+ from services.session_ownership_service import session_ownership_service++ session_ownership_service._session_factory = _lazy_session_factory+ conversation_persistence._session_factory = _lazy_session_factory++ return {+ "document_service": document_service,+ "search_service": search_service,+ "task_service": task_service,+ "chat_service": chat_service,+ "flows_service": flows_service,+ "langflow_file_service": langflow_file_service,+ "auth_service": auth_service,+ "connector_service": connector_service,+ "knowledge_filter_service": knowledge_filter_service,+ "models_service": models_service,+ "monitor_service": monitor_service,+ "session_manager": session_manager,+ "api_key_service": api_key_service,+ "langflow_mcp_service": langflow_mcp_service,+ "docling_service": clients.docling_service,+ "rbac_service": rbac_service,+ "workspace_config_service": workspace_config_service,+ }diff --git a/src/app/factory.py b/src/app/factory.pynew file mode 100644index 000000000..d69aad28a--- /dev/null+++ b/src/app/factory.py@@ -0,0 +1,54 @@+"""FastAPI application factory.++Wires the service container, middleware, routes, and lifespan together+into a ready-to-serve FastAPI app.+"""++from fastapi import FastAPI+from prometheus_fastapi_instrumentator import Instrumentator++from app.container import initialize_services+from app.lifespan import run_shutdown, run_startup+from app.middleware import RequestLoggingMiddleware+from app.routes import register_all_routes+from config.settings import FASTAPI_DEBUG+from utils.logging_config import get_logger+from utils.version_utils import OPENRAG_VERSION++logger = get_logger(__name__)+++async def create_app():+ """Create and configure the FastAPI application"""+ services = await initialize_services()++ app = FastAPI(title="OpenRAG API", version=OPENRAG_VERSION, debug=FASTAPI_DEBUG)+ app.state.services = services+ app.state.background_tasks = set()++ # Wire up ASGI request logging middleware (pure ASGI, not BaseHTTPMiddleware)+ app.add_middleware(RequestLoggingMiddleware)++ try:+ Instrumentator().instrument(app).expose(app)+ except Exception as e:+ logger.error(f"Failed to instrument app with Prometheus: {str(e)}")++ # Register all route handlers and mount the MCP server. The MCP+ # http_app's lifespan context manager is stored on app.state so the+ # application lifespan can enter/exit it at the right time.+ mcp_lifespan_ctx = register_all_routes(app)+ app.state.mcp_lifespan_ctx = mcp_lifespan_ctx++ # Wire startup/shutdown via on_event handlers (not lifespan=) so that+ # `app.router.startup()` / `app.router.shutdown()` triggers them — the+ # integration tests rely on this contract.+ @app.on_event("startup")+ async def _startup():+ await run_startup(app)++ @app.on_event("shutdown")+ async def _shutdown():+ await run_shutdown(app)++ return appdiff --git a/src/app/lifespan.py b/src/app/lifespan.pynew file mode 100644index 000000000..830c861ff--- /dev/null+++ b/src/app/lifespan.py@@ -0,0 +1,270 @@+"""Consolidated startup + shutdown lifecycle for the OpenRAG backend.++`run_startup(app)` and `run_shutdown(app)` collapse what previously lived+as three @app.on_event("startup") handlers and three @app.on_event("shutdown")+handlers into one helper each. The factory registers these as on_event+handlers so they fire under both Starlette's lifespan-from-on_event flow+(production) and `app.router.startup()` / `app.router.shutdown()` (tests).+"""++import asyncio++from fastapi import FastAPI++from config.settings import (+ RBAC_CACHE_BACKEND,+ RBAC_PERMISSION_CACHE_TTL_SECONDS,+ UVICORN_WORKER_COUNT,+ clients,+ get_openrag_config,+)+from services.startup_orchestrator import startup_tasks+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClient++logger = get_logger(__name__)+++async def cleanup_subscriptions_proper(services):+ """Cancel all active webhook subscriptions"""+ logger.info("Cancelling active webhook subscriptions")++ try:+ connector_service = services["connector_service"]+ await connector_service.connection_manager.load_connections()++ all_connections = await connector_service.connection_manager.list_connections()+ active_connections = [+ c for c in all_connections if c.is_active and c.config.get("webhook_channel_id")+ ]++ for connection in active_connections:+ try:+ logger.info(+ "Cancelling subscription for connection",+ connection_id=connection.connection_id,+ )+ connector = await connector_service.get_connector(connection.connection_id)+ if connector:+ subscription_id = connection.config.get("webhook_channel_id")+ await connector.cleanup_subscription(subscription_id)+ logger.info("Cancelled subscription", subscription_id=subscription_id)+ except Exception as e:+ logger.error(+ "Failed to cancel subscription",+ connection_id=connection.connection_id,+ error=str(e),+ )++ logger.info(+ "Finished cancelling subscriptions",+ subscription_count=len(active_connections),+ )++ except Exception as e:+ logger.error("Failed to cleanup subscriptions", error=str(e))+++async def _periodic_backup(services):+ """Run flow backups every 5 minutes once onboarding is complete."""+ while True:+ try:+ await asyncio.sleep(5 * 60)++ config = get_openrag_config()+ if not config.edited:+ logger.debug("Onboarding not completed yet, skipping periodic backup")+ continue++ flows_service = services.get("flows_service")+ if flows_service:+ logger.info("Running periodic flow backup")+ backup_results = await flows_service.backup_all_flows(only_if_changed=True)+ if backup_results["backed_up"]:+ logger.info(+ "Periodic backup completed",+ backed_up=len(backup_results["backed_up"]),+ skipped=len(backup_results["skipped"]),+ )+ else:+ logger.debug(+ "Periodic backup: no flows changed",+ skipped=len(backup_results["skipped"]),+ )+ except asyncio.CancelledError:+ logger.info("Periodic backup task cancelled")+ break+ except Exception as e:+ logger.error(f"Error in periodic backup task: {str(e)}")+++async def run_startup(app: FastAPI):+ """Single source of truth for startup work.++ Expects `app.state.services`, `app.state.mcp_lifespan_ctx`, and+ `app.state.background_tasks` to be set by the factory.+ """+ services = app.state.services+ mcp_lifespan_ctx = getattr(app.state, "mcp_lifespan_ctx", None)++ # Hard-fail if the operator has configured multiple workers. The+ # RBAC permission cache and the OAuth-subject→DB-id cache are+ # both per-process; a second worker silently sees stale+ # permissions for up to OPENRAG_PERM_CACHE_TTL seconds after+ # role mutations. Until the cache moves to a shared backend+ # (Redis), this constraint is real and must be enforced.+ if UVICORN_WORKER_COUNT > 1:+ logger.error(+ "Multi-worker deployment unsupported until cache is "+ "shared across processes. Set UVICORN_WORKERS=1.",+ requested_workers=UVICORN_WORKER_COUNT,+ )+ raise RuntimeError("UVICORN_WORKERS>1 is not supported")+ if RBAC_CACHE_BACKEND not in ("memory",):+ logger.error(+ "Unsupported CACHE_BACKEND. Only 'memory' is wired.",+ requested=RBAC_CACHE_BACKEND,+ )+ raise RuntimeError(f"unsupported CACHE_BACKEND={RBAC_CACHE_BACKEND!r}")+ logger.info(+ "Permission cache configured",+ backend=RBAC_CACHE_BACKEND,+ workers=UVICORN_WORKER_COUNT,+ perm_cache_ttl_s=RBAC_PERMISSION_CACHE_TTL_SECONDS,+ )++ # RBAC kill-switch visibility. OPENRAG_RBAC_ENFORCE=false makes+ # every authenticated user effectively admin — log loudly so+ # operators see it on every boot. Available in all run modes;+ # the operator owns the trade-off.+ from services.rbac_service import is_rbac_enforced+ from utils.run_mode_utils import get_run_mode++ if is_rbac_enforced():+ logger.info("RBAC enforcement is ON", run_mode=get_run_mode())+ else:+ logger.warning(+ "RBAC enforcement is DISABLED — every authenticated "+ "user has full access via the OPENRAG_RBAC_ENFORCE=false "+ "kill switch.",+ run_mode=get_run_mode(),+ )++ # Open the SQL engine on uvicorn's live loop (NOT the one used by+ # asyncio.run(create_app()) which closed already). All RBAC code+ # uses RBACService's lazy factory that reads db.engine.SessionLocal+ # at call time, so it will pick up the binding we set here.+ try:+ from db.engine import init_engine++ init_engine()+ except Exception as e:+ logger.error("DB engine init failed at startup", error=str(e))+ raise++ # One-shot JSON->DB migration + test-fixture cleanup. Runs once+ # per install, idempotent via migration_status rows.+ try:+ from db.engine import SessionLocal as _SL+ from db.migrations_runtime import run as run_runtime_migration++ if _SL is not None:+ async with _SL() as _session:+ await run_runtime_migration(_session)+ await _session.commit()+ except Exception as e:+ logger.error("Runtime DB migration failed", error=str(e))+ raise++ await TelemetryClient.send_event(Category.APPLICATION_STARTUP, MessageId.ORB_APP_STARTED)++ # FastMCP requires its own lifespan to be entered before requests+ # arrive so the StreamableHTTPSessionManager task group exists.+ if mcp_lifespan_ctx:+ await mcp_lifespan_ctx.__aenter__()+ logger.info("FastMCP lifespan started")++ # Start index initialization in background to avoid blocking OIDC endpoints+ t1 = asyncio.create_task(startup_tasks(services))+ app.state.background_tasks.add(t1)+ t1.add_done_callback(app.state.background_tasks.discard)++ # Start periodic task cleanup scheduler+ services["task_service"].start_cleanup_scheduler()++ # Start periodic flow backup task (every 5 minutes)+ backup_task = asyncio.create_task(_periodic_backup(services))+ app.state.background_tasks.add(backup_task)+ backup_task.add_done_callback(app.state.background_tasks.discard)+++async def run_shutdown(app: FastAPI):+ """Single source of truth for shutdown work."""+ services = app.state.services+ mcp_lifespan_ctx = getattr(app.state, "mcp_lifespan_ctx", None)++ logger.info("Application shutdown initiated")++ await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)++ # Cancel and await our long-lived background tasks (startup_tasks,+ # _periodic_backup) before tearing down the resources they touch+ # (clients, task_service, db engine). Without this, a periodic+ # backup can wake up mid-shutdown and crash on a closed connection+ # or, worse, write partial state to a half-disposed engine.+ background_tasks = list(getattr(app.state, "background_tasks", set()))+ for task in background_tasks:+ task.cancel()+ if background_tasks:+ await asyncio.gather(*background_tasks, return_exceptions=True)+ logger.info(+ "Background tasks cancelled at shutdown", count=len(background_tasks)+ )++ # Drain any pending workspace_config DB-mirror tasks before we+ # tear down the engine. Without this, a save_config triggered+ # right before shutdown can be cancelled mid-write, leaving+ # yaml and DB out of sync.+ try:+ wcs = services.get("workspace_config_service") if isinstance(services, dict) else None+ if wcs is not None:+ await wcs.await_pending_mirrors()+ except Exception as e: # noqa: BLE001+ logger.error("Error awaiting pending DB mirrors", error=str(e))++ # Stop FastMCP lifespan+ if mcp_lifespan_ctx:+ try:+ await mcp_lifespan_ctx.__aexit__(None, None, None)+ logger.info("FastMCP lifespan stopped")+ except Exception as e:+ logger.error("Error stopping FastMCP lifespan", error=str(e))++ # Gracefully shutdown OpenSearch connection+ try:+ from utils.opensearch_utils import graceful_opensearch_shutdown++ await graceful_opensearch_shutdown(clients.opensearch)+ except Exception as e:+ logger.error("Error during graceful OpenSearch shutdown", error=str(e))++ await cleanup_subscriptions_proper(services)+ # Cleanup task service (cancels background tasks and process pool)+ await services["task_service"].shutdown()+ # Cleanup async clients (this will also close OpenSearch client if not already closed)+ await clients.cleanup()++ # Dispose the SQL engine cleanly. Previously only the (dead) lifespan+ # path did this; the live shutdown_event leaked the engine.+ try:+ from db.engine import dispose_engine++ await dispose_engine()+ except Exception as e:+ logger.error("Error disposing DB engine", error=str(e))++ # Cleanup telemetry client+ from utils.telemetry.client import cleanup_telemetry_client++ await cleanup_telemetry_client()+ logger.info("Application shutdown completed")diff --git a/src/app/middleware.py b/src/app/middleware.pynew file mode 100644index 000000000..f453499df--- /dev/null+++ b/src/app/middleware.py@@ -0,0 +1,67 @@+"""ASGI middleware for request correlation IDs and structured access logs."""++import time+import uuid++from starlette.types import ASGIApp, Receive, Scope, Send++from utils.logging_config import get_logger++logger = get_logger(__name__)+++class RequestLoggingMiddleware:+ """Pure ASGI middleware that assigns a correlation ID to every request and+ emits a single structured [API] log line with method, path, status code,+ and duration after the response is sent.++ Using pure ASGI (not BaseHTTPMiddleware) so structlog contextvars bound+ inside endpoint handlers propagate back correctly to this middleware.+ """++ def __init__(self, app: ASGIApp) -> None:+ self.app = app++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ if scope["type"] != "http":+ await self.app(scope, receive, send)+ return++ import structlog as _structlog++ _structlog.contextvars.clear_contextvars()++ headers_dict = dict(scope.get("headers", []))+ request_id = headers_dict.get(b"x-request-id", b"").decode() or str(uuid.uuid4())+ path = scope.get("path", "")+ method = scope.get("method", "")++ _structlog.contextvars.bind_contextvars(+ request_id=request_id,+ method=method,+ path=path,+ )++ status_code = 500+ start = time.perf_counter()++ async def send_wrapper(message):+ nonlocal status_code+ if message["type"] == "http.response.start":+ status_code = message["status"]+ headers = list(message.get("headers", []))+ headers.append((b"x-request-id", request_id.encode()))+ message = {**message, "headers": headers}+ await send(message)++ try:+ await self.app(scope, receive, send_wrapper)+ finally:+ duration_ms = round((time.perf_counter() - start) * 1000)+ level = "warning" if status_code >= 500 else "info"+ getattr(logger, level)(+ "[API] Request",+ status_code=status_code,+ duration_ms=duration_ms,+ )+ _structlog.contextvars.clear_contextvars()diff --git a/src/app/routes/__init__.py b/src/app/routes/__init__.pynew file mode 100644index 000000000..92f22a2df--- /dev/null+++ b/src/app/routes/__init__.py@@ -0,0 +1,23 @@+"""Route registration entry point.++The factory calls `register_all_routes(app)`. Each sub-module+owns one cohesive group of endpoints.+"""++from fastapi import FastAPI++from app.routes.internal import register_internal_routes+from app.routes.mcp import mount_mcp+from app.routes.public_v1 import register_public_v1_routes+++def register_all_routes(app: FastAPI):+ """Wire every HTTP/MCP route onto the FastAPI app.++ Returns the MCP http_app's lifespan context manager (or None) so the+ application lifespan can enter/exit it at the right time.+ """+ register_internal_routes(app)+ register_public_v1_routes(app)+ mcp_lifespan_ctx = mount_mcp(app)+ return mcp_lifespan_ctxdiff --git a/src/app/routes/internal.py b/src/app/routes/internal.pynew file mode 100644index 000000000..78bc8ec63--- /dev/null+++ b/src/app/routes/internal.py@@ -0,0 +1,405 @@+"""Internal v0 route registrations.++All endpoints not under /v1/* and not the MCP mount.++Auth and service injection are handled inside each handler via FastAPI+Depends, not here. This module only wires URL → handler.+"""++from fastapi import FastAPI++from api import (+ auth,+ chat,+ connectors,+ docling,+ documents,+ flows,+ knowledge_filter,+ langflow_files,+ models,+ nudges,+ oidc,+ provider_health,+ router,+ search,+ settings,+ tasks,+ upload,+)+from api import keys as api_keys+from api.health import health_check, opensearch_health_ready+from connectors.aws_s3.api import (+ s3_bucket_status,+ s3_configure,+ s3_defaults,+ s3_list_buckets,+)+from connectors.ibm_cos.api import (+ ibm_cos_bucket_status,+ ibm_cos_configure,+ ibm_cos_defaults,+ ibm_cos_list_buckets,+)+++def register_internal_routes(app: FastAPI):+ # Langflow Files endpoints+ app.add_api_route(+ "/langflow/files/upload",+ langflow_files.upload_user_file,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/langflow/ingest",+ langflow_files.run_ingestion,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/langflow/files",+ langflow_files.delete_user_files,+ methods=["DELETE"],+ tags=["internal"],+ )+ app.add_api_route(+ "/langflow/upload_ingest",+ langflow_files.upload_and_ingest_user_file,+ methods=["POST"],+ tags=["internal"],+ )++ # Upload endpoints+ app.add_api_route("/upload_context", upload.upload_context, methods=["POST"], tags=["internal"])+ app.add_api_route("/upload_path", upload.upload_path, methods=["POST"], tags=["internal"])+ app.add_api_route("/upload_options", upload.upload_options, methods=["GET"], tags=["internal"])+ app.add_api_route("/upload_bucket", upload.upload_bucket, methods=["POST"], tags=["internal"])++ # Task endpoints+ app.add_api_route("/tasks/{task_id}", tasks.task_status, methods=["GET"], tags=["internal"])+ app.add_api_route("/tasks", tasks.all_tasks, methods=["GET"], tags=["internal"])+ app.add_api_route(+ "/tasks/{task_id}/cancel",+ tasks.cancel_task,+ methods=["POST"],+ tags=["internal"],+ )++ # Search endpoint+ app.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])++ # Knowledge Filter endpoints+ app.add_api_route(+ "/knowledge-filter",+ knowledge_filter.create_knowledge_filter,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/search",+ knowledge_filter.search_knowledge_filters,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/{filter_id}",+ knowledge_filter.get_knowledge_filter,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/{filter_id}",+ knowledge_filter.update_knowledge_filter,+ methods=["PUT"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/{filter_id}",+ knowledge_filter.delete_knowledge_filter,+ methods=["DELETE"],+ tags=["internal"],+ )++ # Knowledge Filter Subscription endpoints+ app.add_api_route(+ "/knowledge-filter/{filter_id}/subscribe",+ knowledge_filter.subscribe_to_knowledge_filter,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/{filter_id}/subscriptions",+ knowledge_filter.list_knowledge_filter_subscriptions,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/knowledge-filter/{filter_id}/subscribe/{subscription_id}",+ knowledge_filter.cancel_knowledge_filter_subscription,+ methods=["DELETE"],+ tags=["internal"],+ )++ # Knowledge Filter Webhook endpoint (no auth required - called by OpenSearch)+ app.add_api_route(+ "/knowledge-filter/{filter_id}/webhook/{subscription_id}",+ knowledge_filter.knowledge_filter_webhook,+ methods=["POST"],+ tags=["internal"],+ )++ # Chat endpoints+ app.add_api_route("/chat", chat.chat_endpoint, methods=["POST"], tags=["internal"])+ app.add_api_route("/langflow", chat.langflow_endpoint, methods=["POST"], tags=["internal"])++ # Chat history endpoints+ app.add_api_route(+ "/chat/history", chat.chat_history_endpoint, methods=["GET"], tags=["internal"]+ )+ app.add_api_route(+ "/langflow/history",+ chat.langflow_history_endpoint,+ methods=["GET"],+ tags=["internal"],+ )++ # Session deletion endpoint+ app.add_api_route(+ "/sessions/{session_id}",+ chat.delete_session_endpoint,+ methods=["DELETE"],+ tags=["internal"],+ )++ # Authentication endpoints+ app.add_api_route("/auth/init", auth.auth_init, methods=["POST"], tags=["internal"])+ app.add_api_route("/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"])+ app.add_api_route("/auth/me", auth.auth_me, methods=["GET"], tags=["internal"])+ app.add_api_route("/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"])+ app.add_api_route("/auth/ibm/login", auth.ibm_login, methods=["POST"], tags=["internal"])++ # Connector endpoints+ app.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])+ # IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route(+ "/connectors/ibm_cos/defaults",+ ibm_cos_defaults,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/ibm_cos/configure",+ ibm_cos_configure,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/ibm_cos/{connection_id}/buckets",+ ibm_cos_list_buckets,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/ibm_cos/{connection_id}/bucket-status",+ ibm_cos_bucket_status,+ methods=["GET"],+ tags=["internal"],+ )+ # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route(+ "/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"]+ )+ app.add_api_route(+ "/connectors/aws_s3/configure",+ s3_configure,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/aws_s3/{connection_id}/buckets",+ s3_list_buckets,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/aws_s3/{connection_id}/bucket-status",+ s3_bucket_status,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/{connector_type}/sync",+ connectors.connector_sync,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/sync-all",+ connectors.sync_all_connectors,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/{connector_type}/status",+ connectors.connector_status,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/{connector_type}/token",+ connectors.connector_token,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/{connector_type}/disconnect",+ connectors.connector_disconnect,+ methods=["DELETE"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/{connector_type}/webhook",+ connectors.connector_webhook,+ methods=["POST", "GET"],+ tags=["internal"],+ )++ # Document endpoints+ app.add_api_route(+ "/documents/check-filename",+ documents.check_filename_exists,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/documents/delete-by-filename",+ documents.delete_documents_by_filename,+ methods=["POST"],+ tags=["internal"],+ )++ # OIDC endpoints+ app.add_api_route(+ "/.well-known/openid-configuration",+ oidc.oidc_discovery,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route("/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"])+ app.add_api_route(+ "/auth/introspect",+ oidc.token_introspection,+ methods=["POST"],+ tags=["internal"],+ )++ # Settings endpoints+ app.add_api_route("/settings", settings.get_settings, methods=["GET"], tags=["internal"])+ app.add_api_route("/settings", settings.update_settings, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/onboarding/state",+ settings.update_onboarding_state,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/openrag-docs/refresh",+ settings.refresh_openrag_docs,+ methods=["POST"],+ tags=["internal"],+ )++ # Provider health check endpoint+ app.add_api_route(+ "/provider/health",+ provider_health.check_provider_health,+ methods=["GET"],+ tags=["internal"],+ )++ # Health check endpoints+ app.add_api_route("/health", health_check, methods=["GET"], tags=["internal"])+ app.add_api_route("/search/health", opensearch_health_ready, methods=["GET"], tags=["internal"])++ # Models endpoints+ app.add_api_route(+ "/models/openai", models.get_openai_models, methods=["POST"], tags=["internal"]+ )+ app.add_api_route(+ "/models/anthropic",+ models.get_anthropic_models,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/models/ollama", models.get_ollama_models, methods=["GET"], tags=["internal"]+ )+ app.add_api_route("/models/ibm", models.get_ibm_models, methods=["POST"], tags=["internal"])++ # Onboarding endpoints+ app.add_api_route("/onboarding", settings.onboarding, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/onboarding/rollback",+ settings.rollback_onboarding,+ methods=["POST"],+ tags=["internal"],+ )++ # Docling preset update endpoint+ app.add_api_route(+ "/settings/docling-preset",+ settings.update_docling_preset,+ methods=["PATCH"],+ tags=["internal"],+ )++ # Nudges endpoints+ app.add_api_route(+ "/nudges", nudges.nudges_from_kb_endpoint, methods=["POST"], tags=["internal"]+ )+ app.add_api_route(+ "/nudges/{chat_id}",+ nudges.nudges_from_chat_id_endpoint,+ methods=["POST"],+ tags=["internal"],+ )++ # Flow reset endpoint+ app.add_api_route(+ "/reset-flow/{flow_type}",+ flows.reset_flow_endpoint,+ methods=["POST"],+ tags=["internal"],+ )++ # Router upload ingest endpoint+ app.add_api_route(+ "/router/upload_ingest",+ router.upload_ingest_router,+ methods=["POST"],+ tags=["internal"],+ )++ # Docling service proxy+ app.add_api_route("/docling/health", docling.health, methods=["GET"], tags=["internal"])++ # ===== Users / RBAC Endpoints (JWT auth) =====+ from api import config as config_api+ from api import users as users_api+ from api.admin import rbac as admin_rbac++ app.include_router(users_api.router)+ app.include_router(admin_rbac.router)+ # Public — must work pre-auth so the onboarding wizard can render.+ app.include_router(config_api.router)++ # ===== API Key Management Endpoints (JWT auth for UI) =====+ app.add_api_route("/keys", api_keys.list_keys_endpoint, methods=["GET"], tags=["internal"])+ app.add_api_route("/keys", api_keys.create_key_endpoint, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/keys/{key_id}",+ api_keys.revoke_key_endpoint,+ methods=["DELETE"],+ tags=["internal"],+ )diff --git a/src/app/routes/mcp.py b/src/app/routes/mcp.pynew file mode 100644index 000000000..3922829b8--- /dev/null+++ b/src/app/routes/mcp.py@@ -0,0 +1,29 @@+"""FastMCP streamable-HTTP server mounted at /mcp.++Returns the MCP http_app's lifespan context manager so the application+lifespan can enter/exit it inline (FastAPI does not propagate lifespan+to mounted sub-apps automatically).+"""++from fastapi import FastAPI++from mcp_http.server import create_mcp_server+from utils.logging_config import get_logger++logger = get_logger(__name__)+++def mount_mcp(app: FastAPI):+ """Mount the FastMCP app at /mcp and return its lifespan context manager."""+ logger.info("Creating MCP server")+ mcp_server = create_mcp_server(app)+ mcp_http_app = mcp_server.http_app(transport="streamable-http", path="/")+ app.mount("/mcp", mcp_http_app)+ logger.info("MCP server mounted at /mcp (streamable-http)")++ # FastMCP requires its own lifespan to be run so that the+ # StreamableHTTPSessionManager task group is initialized before requests arrive.+ # FastAPI does not automatically propagate lifespan to mounted sub-apps,+ # so the application lifespan enters/exits this context manager directly.+ mcp_lifespan_ctx = mcp_http_app.router.lifespan_context(mcp_http_app)+ return mcp_lifespan_ctxdiff --git a/src/app/routes/public_v1.py b/src/app/routes/public_v1.pynew file mode 100644index 000000000..d7e735fb9--- /dev/null+++ b/src/app/routes/public_v1.py@@ -0,0 +1,118 @@+"""Public /v1/* route registrations (API-key auth)."""++from fastapi import FastAPI++from api.v1 import (+ chat as v1_chat,+)+from api.v1 import (+ documents as v1_documents,+)+from api.v1 import (+ knowledge_filters as v1_knowledge_filters,+)+from api.v1 import (+ models as v1_models,+)+from api.v1 import (+ search as v1_search,+)+from api.v1 import (+ settings as v1_settings,+)+++def register_public_v1_routes(app: FastAPI):++ # Chat endpoints+ app.add_api_route("/v1/chat", v1_chat.chat_create_endpoint, methods=["POST"], tags=["public"])+ app.add_api_route("/v1/chat", v1_chat.chat_list_endpoint, methods=["GET"], tags=["public"])+ app.add_api_route(+ "/v1/chat/{chat_id}",+ v1_chat.chat_get_endpoint,+ methods=["GET"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/chat/{chat_id}",+ v1_chat.chat_delete_endpoint,+ methods=["DELETE"],+ tags=["public"],+ )++ # Search endpoint+ app.add_api_route("/v1/search", v1_search.search_endpoint, methods=["POST"], tags=["public"])++ # Documents endpoints+ app.add_api_route(+ "/v1/documents/ingest",+ v1_documents.ingest_endpoint,+ methods=["POST"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/tasks/{task_id}",+ v1_documents.task_status_endpoint,+ methods=["GET"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/documents",+ v1_documents.delete_document_endpoint,+ methods=["DELETE"],+ tags=["public"],+ )++ # Settings endpoints+ app.add_api_route(+ "/v1/settings",+ v1_settings.get_settings_endpoint,+ methods=["GET"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/settings",+ v1_settings.update_settings_endpoint,+ methods=["POST"],+ tags=["public"],+ )++ # Models endpoint+ app.add_api_route(+ "/v1/models/{provider}",+ v1_models.list_models_endpoint,+ methods=["GET"],+ tags=["public"],+ )++ # Knowledge filters endpoints+ app.add_api_route(+ "/v1/knowledge-filters",+ v1_knowledge_filters.create_endpoint,+ methods=["POST"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/knowledge-filters/search",+ v1_knowledge_filters.search_endpoint,+ methods=["POST"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/knowledge-filters/{filter_id}",+ v1_knowledge_filters.get_endpoint,+ methods=["GET"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/knowledge-filters/{filter_id}",+ v1_knowledge_filters.update_endpoint,+ methods=["PUT"],+ tags=["public"],+ )+ app.add_api_route(+ "/v1/knowledge-filters/{filter_id}",+ v1_knowledge_filters.delete_endpoint,+ methods=["DELETE"],+ tags=["public"],+ )diff --git a/src/config/settings.py b/src/config/settings.pyindex 59530aafa..abdf51e6a 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -1,9 +1,7 @@-from config.paths import get_flows_pathimport asyncio+import concurrent.futuresimport osimport threading-import concurrent.futures-from utils.env_utils import get_env_int, get_env_floatimport httpxfrom agentd.patch import patch_openai_with_mcp@@ -11,11 +9,14 @@ from dotenv import load_dotenvfrom openai import AsyncOpenAIfrom opensearchpy import AsyncOpenSearchfrom opensearchpy._async.http_aiohttp import AIOHttpConnection-from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL-from utils.container_utils import get_container_host, determine_docling_host+from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL+from config.paths import get_flows_path+from utils.container_utils import determine_docling_host, get_container_hostfrom utils.embedding_fields import build_knn_vector_field+from utils.env_utils import get_env_float, get_env_intfrom utils.logging_config import get_logger+# Import configuration managerfrom .config_manager import config_manager@@ -51,6 +52,10 @@ LANGFLOW_SUPERUSER_PASSWORD = os.getenv("LANGFLOW_SUPERUSER_PASSWORD")# Allow explicit key via environment; generation will be skipped if setLANGFLOW_KEY = os.getenv("LANGFLOW_KEY")SESSION_SECRET = os.getenv("SESSION_SECRET", "your-secret-key-change-in-production")+# Optional explicit JWT signing key. When set (and IBM auth is off),+# RSA keypair generation is skipped. Read here so callers don't poke+# os.environ directly.+JWT_SIGNING_KEY = os.getenv("JWT_SIGNING_KEY")GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID")GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET")@@ -63,6 +68,41 @@ DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY", "")ENVIRONMENT = os.getenv("ENVIRONMENT", "")+# Enable FastAPI's `debug` mode (verbose tracebacks in HTTP error responses+# on the FastAPI app instance). Named explicitly so it isn't confused with+# logging-level "debug" or other unrelated debug flags.+#+# Default behavior:+# * If FASTAPI_DEBUG is set explicitly (true/false), that wins.+# * Otherwise, defaults to True when LOG_LEVEL=DEBUG (developer is already+# opting into verbose output), False otherwise. This gives `LOG_LEVEL=DEBUG`+# in .env a single-knob "dev mode" effect without forcing it on in prod.+_fastapi_debug_default = "true" if os.getenv("LOG_LEVEL", "INFO").upper() == "DEBUG" else "false"+FASTAPI_DEBUG = os.getenv("FASTAPI_DEBUG", _fastapi_debug_default).lower() in (+ "true",+ "1",+ "yes",+)++# Whether uvicorn emits an access log line per HTTP request. On by+# default; flip via ACCESS_LOG=false (e.g. when fronted by a load balancer+# that already logs requests, or to reduce log noise in CI).+ACCESS_LOG_ENABLED = os.getenv("ACCESS_LOG", "true").lower() in ("true", "1", "yes")++# Number of uvicorn worker processes to allow. Multi-worker is currently+# unsupported because the RBAC permission cache and the OAuth-subject→DB-id+# cache are per-process; the lifespan startup hook hard-fails if this is >1+# until the cache moves to a shared backend (Redis).+UVICORN_WORKER_COUNT = get_env_int("UVICORN_WORKERS", 1)++# Backend for the in-process RBAC permission cache. Only "memory" is wired+# today; the lifespan hook rejects anything else.+RBAC_CACHE_BACKEND = os.getenv("CACHE_BACKEND", "memory").lower()++# TTL (seconds) for cached RBAC permission lookups. Stale permissions can+# linger for up to this many seconds after a role mutation.+RBAC_PERMISSION_CACHE_TTL_SECONDS = get_env_int("OPENRAG_PERM_CACHE_TTL", 60)+# Docling service URL configuration# Priority:# 1. DOCLING_SERVE_URL environment variable@@ -537,7 +577,7 @@ class AppClients:)logger.info(f"HTTP/2 probe successful with {model_name}")return True- except (asyncio.TimeoutError, Exception) as probe_error:+ except (TimeoutError, Exception) as probe_error:logger.warning(f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1", error=str(probe_error))return Falsefinally:@@ -587,7 +627,7 @@ class AppClients:logger.info("Successfully initialized OpenAI client")except Exception as e:logger.error(f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}")- raise ValueError(f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable.")+ raise ValueError(f"Failed to initialize OpenAI client: {str(e)}. Please complete onboarding or set OPENAI_API_KEY environment variable.") from ereturn self._patched_async_clientdiff --git a/src/main.py b/src/main.pyindex 57b7a56c7..6042f460c 100644--- a/src/main.py+++ b/src/main.py@@ -1,2461 +1,71 @@+"""OpenRAG backend entry point.++Most of what used to live in this file moved into focused modules:+- app/factory.py — FastAPI app construction+- app/container.py — service DI container+- app/lifespan.py — startup + shutdown lifecycle+- app/middleware.py — request logging+- app/routes/ — route registrars (internal, public_v1, mcp)+- services/default_docs_service.py — bundled-docs onboarding/refresh+- services/startup_orchestrator.py — per-startup tasks (OpenSearch, flows, MCP URLs)+- utils/opensearch_init.py — OpenSearch index/security setup+- utils/jwt_keygen.py — RSA keypair generation+- utils/url_content_fetcher.py — URL → text-file helper+- api/health.py — liveness + readiness probes++This module is intentionally thin: import bootstrap, run Alembic, build+the app, run uvicorn. The re-exports below preserve the+`from main import …` contract used by tests and api/settings.py.+"""+import bootstrap # noqa: F401 — must be first; loads .env + structured logging-from contextlib import asynccontextmanager-from utils.version_utils import OPENRAG_VERSION+import asyncioimport atexit-import hashlib-import html-import httpx-import os-import re-import subprocess-import tempfile-import time-import uuid-from html.parser import HTMLParser--# Configure structured logging early-from connectors.langflow_connector_service import LangflowConnectorService-from connectors.service import ConnectorService-from services.flows_service import FlowsService-from utils.embeddings import create_index_body-from utils.logging_config import get_logger-from utils.encryption import enforce_startup_prerequisites-from utils.telemetry import TelemetryClient, Category, MessageId-from fastapi import FastAPI, Request-from fastapi.responses import JSONResponse-from prometheus_fastapi_instrumentator import Instrumentator-from starlette.types import ASGIApp, Receive, Scope, Send---# API endpoints-from api import (- auth,- chat,- connectors,- docling,- documents,- flows,- knowledge_filter,- langflow_files,- models,- nudges,- oidc,- provider_health,- router,- search,- settings,- tasks,- upload,-)-from api.connector_router import ConnectorRouter-from connectors.ibm_cos.api import (- ibm_cos_defaults,- ibm_cos_configure,- ibm_cos_list_buckets,- ibm_cos_bucket_status,-)-from connectors.aws_s3.api import (- s3_defaults,- s3_configure,- s3_list_buckets,- s3_bucket_status,+# Re-exported so tests that monkeypatch "main.httpx.AsyncClient" keep working.+# Module attributes in Python are shared singletons, so the patch propagates+# to services/default_docs_service.py which calls httpx.AsyncClient directly.+import httpx # noqa: F401++from app.factory import create_app+from config.settings import ACCESS_LOG_ENABLED+from services.default_docs_service import (+ _get_remote_docs_signature,+ _should_use_url_default_docs_ingest,+ ingest_default_documents_when_ready,+ refresh_default_openrag_docs,)-from services.api_key_service import APIKeyService-from api import keys as api_keys-from api.v1 import (- chat as v1_chat,- search as v1_search,- documents as v1_documents,- settings as v1_settings,- models as v1_models,- knowledge_filters as v1_knowledge_filters,-)-from mcp_http.server import create_mcp_server--# Configuration and setup-from config.settings import (- DEFAULT_DOCS_CRAWL_DEPTH,- DEFAULT_DOCS_INGEST_SOURCE,- DEFAULT_DOCS_URL,- API_KEYS_INDEX_BODY,- API_KEYS_INDEX_NAME,- DISABLE_INGEST_WITH_LANGFLOW,- FETCH_OPENRAG_DOCS_AT_STARTUP,- INGESTION_TIMEOUT,- INDEX_BODY,- LANGFLOW_URL_INGEST_FLOW_ID,- SESSION_SECRET,- clients,- config_manager,- get_embedding_model,- get_index_name,- is_no_auth_mode,- get_openrag_config,-)-from services.auth_service import AuthService-from services.langflow_mcp_service import LangflowMCPService-from services.chat_service import ChatService--# Services-from services.document_service import DocumentService-from services.knowledge_filter_service import KnowledgeFilterService--# Configuration and setup-# Services-from services.langflow_file_service import LangflowFileService-from services.models_service import ModelsService-from services.monitor_service import MonitorService-from services.search_service import SearchService, register_search_service-from services.task_service import TaskService-from session_manager import SessionManager+from services.startup_orchestrator import startup_tasks+from utils.encryption import enforce_startup_prerequisites+from utils.jwt_keygen import generate_jwt_keys+from utils.logging_config import get_logger+from utils.opensearch_init import _ensure_opensearch_index, init_indexenforce_startup_prerequisites()logger = get_logger(__name__)-# Files to exclude from startup ingestion-EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}-URL_INGEST_EXCLUDED_INGESTION_FILES = {"openrag-documentation.pdf"}---class _VisibleTextHTMLParser(HTMLParser):- """Extract visible text while skipping script/style content."""-- def __init__(self):- super().__init__(convert_charrefs=True)- self._ignored_depth = 0- self._chunks: list[str] = []-- def handle_starttag(self, tag, attrs):- if tag.lower() in {"script", "style"}:- self._ignored_depth += 1-- def handle_endtag(self, tag):- if tag.lower() in {"script", "style"} and self._ignored_depth > 0:- self._ignored_depth -= 1-- def handle_data(self, data):- if self._ignored_depth == 0 and data and not data.isspace():- self._chunks.append(data)-- def get_text(self) -> str:- return " ".join(self._chunks)---async def wait_for_opensearch(opensearch_client=None):- """Wait for OpenSearch to be ready, delegating to the shared utility."""- from utils.opensearch_utils import (- wait_for_opensearch as _wait_for_opensearch,- OpenSearchNotReadyError,- )-- try:- await _wait_for_opensearch(opensearch_client or clients.opensearch)- await TelemetryClient.send_event(- Category.OPENSEARCH_SETUP, MessageId.ORB_OS_CONN_ESTABLISHED- )- except OpenSearchNotReadyError:- await TelemetryClient.send_event(- Category.OPENSEARCH_SETUP, MessageId.ORB_OS_TIMEOUT- )- raise---async def configure_alerting_security():- """Configure OpenSearch alerting plugin security settings"""- try:- # For testing, disable backend role filtering to allow all authenticated users- # In production, you'd want to configure proper roles instead- alerting_settings = {- "persistent": {- "plugins.alerting.filter_by_backend_roles": "false",- "opendistro.alerting.filter_by_backend_roles": "false",- "opensearch.notifications.general.filter_by_backend_roles": "false",- }- }-- # Use admin client (clients.opensearch uses admin credentials)- response = await clients.opensearch.cluster.put_settings(body=alerting_settings)- logger.info(- "Alerting security settings configured successfully", response=response- )- except Exception as e:- logger.error("Failed to configure alerting security settings", error=str(e))- # Don't fail startup if alerting config fails---async def _ensure_opensearch_index():- """Ensure OpenSearch index exists when using traditional connector service."""- try:- index_name = get_index_name()- # Check if index already exists- if await clients.opensearch.indices.exists(index=index_name):- logger.info("[OPENSEARCH] Index already exists", index_name=index_name)- return-- # Create the index with hard-coded INDEX_BODY (uses OpenAI embedding dimensions)- await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)- logger.info(- "Created OpenSearch index for traditional connector service",- index_name=index_name,- vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"][- "dimension"- ],- )- await TelemetryClient.send_event(- Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED- )-- except Exception as e:- logger.error(- "Failed to initialize OpenSearch index for traditional connector service",- error=str(e),- index_name=get_index_name(),- )- await TelemetryClient.send_event(- Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL- )- # Don't raise the exception to avoid breaking the initialization- # The service can still function, document operations might fail later---async def init_index(opensearch_client=None, admin_username: str = None):- """Initialize OpenSearch index and security roles"""- os_client = opensearch_client or clients.opensearch- try:- await wait_for_opensearch(opensearch_client)-- # Initialize OpenSearch security configuration (roles and mapping)- from utils.opensearch_utils import setup_opensearch_security- await setup_opensearch_security(os_client, admin_username=admin_username)-- # Get the configured embedding model from user configuration- config = get_openrag_config()- embedding_model = config.knowledge.embedding_model-- # Create index body- index_body = await create_index_body()-- # Create documents index- index_name = get_index_name()- if not await os_client.indices.exists(index=index_name):- await os_client.indices.create(index=index_name, body=index_body)- logger.info(- "Created OpenSearch index",- index_name=index_name,- embedding_model=embedding_model,- )- await TelemetryClient.send_event(- Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED- )- else:- logger.info(- "Index already exists, skipping creation and changing number of replicas",- index_name=index_name,- embedding_model=embedding_model,- )- # Set number of replicas to 0 to not create unused nodes in OpenSearch, in case it was created with more replicas- current = await os_client.indices.get_settings(index=index_name)- current_replicas = int(- current[index_name]["settings"]["index"].get("number_of_replicas", 1)- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info(- "Updated documents index settings",- )- await TelemetryClient.send_event(- Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS- )-- # Create knowledge filters index- knowledge_filter_index_name = "knowledge_filters"- knowledge_filter_index_body = {- "settings": {- "index": {"number_of_replicas": 0, "number_of_shards": 1},- },- "mappings": {- "properties": {- "id": {"type": "keyword"},- "name": {"type": "text", "analyzer": "standard"},- "description": {"type": "text", "analyzer": "standard"},- "query_data": {"type": "text"}, # Store as text for searching- "owner": {"type": "keyword"},- "allowed_users": {"type": "keyword"},- "allowed_groups": {"type": "keyword"},- "subscriptions": {"type": "object"}, # Store subscription data- "created_at": {"type": "date"},- "updated_at": {"type": "date"},- }- },- }-- if not await os_client.indices.exists(index=knowledge_filter_index_name):- await os_client.indices.create(- index=knowledge_filter_index_name, body=knowledge_filter_index_body- )- logger.info(- "Created knowledge filters index",- index_name=knowledge_filter_index_name,- )- await TelemetryClient.send_event(- Category.OPENSEARCH_INDEX, MessageId.ORB_OS_KF_INDEX_CREATED- )- else:- logger.info(- "Knowledge filters index already exists, skipping creation",- index_name=knowledge_filter_index_name,- )-- current = await os_client.indices.get_settings(index=knowledge_filter_index_name)- current_replicas = int(- current[knowledge_filter_index_name]["settings"]["index"].get("number_of_replicas", 1)- )- if current_replicas != 0:- await os_client.indices.put_settings(- index=knowledge_filter_index_name,- body={"index": {"number_of_replicas": 0}},- )- logger.info(- "Updated knowledge filters index settings",- )-- # Create API keys index for public API authentication- if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):- await os_client.indices.create(- index=API_KEYS_INDEX_NAME, body=API_KEYS_INDEX_BODY- )- logger.info("Created API keys index", index_name=API_KEYS_INDEX_NAME)- else:- logger.info(- "API keys index already exists, skipping creation",- index_name=API_KEYS_INDEX_NAME,- )-- # Configure alerting plugin security settings (admin-level).- # Ensure the global OpenSearch client used by alerting points to the- # same authenticated/admin-capable client selected above (including IBM mode).- await configure_alerting_security()-- except Exception as e:- error_msg = str(e).lower()- if "disk usage exceeded" in error_msg or "flood-stage watermark" in error_msg:- logger.error(- "OpenSearch disk usage exceeded flood-stage watermark. Index creation failed."- )- raise Exception(- "OpenSearch disk space is full (flood-stage watermark exceeded). "- "Please free up disk space on your Docker volume or host machine to continue."- ) from e- raise e--def generate_jwt_keys():- """Generate RSA keys for JWT signing if they don't exist"""- from config.paths import get_keys_path- keys_dir = get_keys_path()- private_key_path = os.path.join(keys_dir, "private_key.pem")- public_key_path = os.path.join(keys_dir, "public_key.pem")-- # Create keys directory if it doesn't exist- os.makedirs(keys_dir, exist_ok=True)-- # Generate keys if they don't exist- if not os.path.exists(private_key_path):- try:- # Generate private key- subprocess.run(- ["openssl", "genrsa", "-out", private_key_path, "2048"],- check=True,- capture_output=True,- )-- # Set restrictive permissions on private key (readable by owner only)- os.chmod(private_key_path, 0o600)-- # Generate public key- subprocess.run(- [- "openssl",- "rsa",- "-in",- private_key_path,- "-pubout",- "-out",- public_key_path,- ],- check=True,- capture_output=True,- )-- # Set permissions on public key (readable by all)- os.chmod(public_key_path, 0o644)-- logger.info("Generated RSA keys for JWT signing")- except subprocess.CalledProcessError as e:- logger.error("Failed to generate RSA keys", error=str(e))- TelemetryClient.send_event_sync(- Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_JWT_KEY_FAIL- )- raise- else:- logger.info("RSA keys already exist")---def _get_documents_dir():- """Get the documents directory path, handling both Docker and local environments."""- from config.paths import get_documents_path- path = get_documents_path()- logger.debug(f"Using documents path: {path}")- return path---def _should_use_url_default_docs_ingest() -> bool:- """Return whether default docs ingestion should use URL crawling."""- return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)---async def ingest_openrag_docs_when_ready(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- jwt_token=None,-):- """Ingest OpenRAG docs during onboarding."""- use_url_ingest = _should_use_url_default_docs_ingest()- task_id = None- if use_url_ingest:- try:- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START- )- if DISABLE_INGEST_WITH_LANGFLOW:- task_id = await _ingest_default_documents_url(- document_service=document_service,- models_service=models_service,- docs_url=DEFAULT_DOCS_URL,- crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,- jwt_token=jwt_token,- )- else:- logger.info(- "Ingesting default documents using Langflow",- docs_url=DEFAULT_DOCS_URL,- )- task_id = await _ingest_default_documents_url_langflow(- langflow_file_service=langflow_file_service,- session_manager=session_manager,- task_service=task_service,- docs_url=DEFAULT_DOCS_URL,- crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,- jwt_token=jwt_token,- )- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_COMPLETE- )- except Exception as e:- logger.error("Default URL documents ingestion failed", error=str(e))- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED- )- return task_id---async def ingest_default_documents_when_ready(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- jwt_token=None,-):- """Ingest default OpenRAG docs during onboarding."""- try:- logger.info(- "Ingesting default documents when ready",- disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,- ingest_source=DEFAULT_DOCS_INGEST_SOURCE,- )- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START- )- task_id = await ingest_openrag_docs_when_ready(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- jwt_token=jwt_token,- )-- base_dir = _get_documents_dir()- if not os.path.isdir(base_dir):- raise FileNotFoundError(- f"Default documents directory not found: {base_dir}"- )-- excluded_files = set(EXCLUDED_INGESTION_FILES)- if _should_use_url_default_docs_ingest():- excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)-- file_paths = [- os.path.join(root, fn)- for root, _, files in os.walk(base_dir)- for fn in files- if fn not in excluded_files- ]-- if not file_paths:- raise FileNotFoundError(f"No default documents found in {base_dir}")-- if DISABLE_INGEST_WITH_LANGFLOW:- new_task_id = await _ingest_default_documents_openrag(- document_service,- models_service,- task_service,- file_paths,- existing_task_id=task_id,- connector_type="local",- jwt_token=jwt_token,- )- task_id = new_task_id or task_id- else:- new_task_id = await _ingest_default_documents_langflow(- langflow_file_service,- session_manager,- task_service,- file_paths,- existing_task_id=task_id,- connector_type="local",- jwt_token=jwt_token,- )- task_id = new_task_id or task_id-- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE- )-- return task_id-- except Exception as e:- logger.error("Default documents ingestion failed", error=str(e))- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED- )- raise---async def _ingest_default_documents_langflow(- langflow_file_service,- session_manager,- task_service,- file_paths,- existing_task_id: str = None,- connector_type: str = "openrag_docs",- jwt_token=None,-):- """Ingest default documents using Langflow upload-ingest-delete pipeline."""-- logger.info(- "Using Langflow ingestion pipeline for default documents",- file_count=len(file_paths),- )-- from session_manager import AnonymousUser-- anonymous_user = AnonymousUser()- effective_jwt = jwt_token-- if not effective_jwt and session_manager:- session_manager.get_user_opensearch_client(- anonymous_user.user_id, effective_jwt- )- if hasattr(session_manager, "_anonymous_jwt"):- effective_jwt = session_manager._anonymous_jwt-- # Prepare tweaks for default documents with anonymous user metadata- default_tweaks = {- "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {- "docs_metadata": [- {"key": "owner", "value": None},- {"key": "owner_name", "value": anonymous_user.name},- {"key": "owner_email", "value": anonymous_user.email},- {"key": "connector_type", "value": "openrag_docs"},- {"key": "is_sample_data", "value": "true"},- ]- }- }-- # Create a langflow upload task for trackable progress- task_id = await task_service.create_langflow_upload_task(- user_id=None, # Anonymous user- file_paths=file_paths,- langflow_file_service=langflow_file_service,- session_manager=session_manager,- jwt_token=effective_jwt,- owner_name=anonymous_user.name,- owner_email=anonymous_user.email,- session_id=None, # No session for default documents- tweaks=default_tweaks,- settings=None, # Use default ingestion settings- replace_duplicates=True,- connector_type=connector_type,- existing_task_id=existing_task_id,- )-- logger.info(- "Started Langflow ingestion task for default documents",- task_id=task_id,- file_count=len(file_paths),- )- return task_id---async def _ingest_default_documents_url_langflow(- langflow_file_service,- session_manager,- task_service,- docs_url: str,- crawl_depth: int,- jwt_token=None,-):- """Ingest default URL docs using the Langflow URL ingestion pipeline."""- if not docs_url:- raise ValueError("DEFAULT_DOCS_URL is not configured")-- logger.info(- "Using Langflow URL ingestion pipeline for default documents",- docs_url=docs_url,- crawl_depth=crawl_depth,- )-- from session_manager import AnonymousUser-- anonymous_user = AnonymousUser()- effective_jwt = jwt_token-- if not effective_jwt and session_manager:- session_manager.get_user_opensearch_client(- anonymous_user.user_id, effective_jwt- )- if hasattr(session_manager, "_anonymous_jwt"):- effective_jwt = session_manager._anonymous_jwt-- default_tweaks = {- "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {- "docs_metadata": [- {"key": "owner", "value": None},- {"key": "owner_name", "value": anonymous_user.name},- {"key": "owner_email", "value": anonymous_user.email},- {"key": "connector_type", "value": "openrag_docs"},- {"key": "is_sample_data", "value": "true"},- ]- }- }-- task_id = await task_service.create_langflow_url_upload_task(- owner_user_id=None,- docs_url=docs_url,- crawl_depth=crawl_depth,- langflow_file_service=langflow_file_service,- session_manager=session_manager,- jwt_token=effective_jwt,- owner_name=anonymous_user.name,- owner_email=anonymous_user.email,- connector_type="openrag_docs",- tweaks=default_tweaks,- )-- logger.info(- "Started Langflow URL ingestion task for default documents",- task_id=task_id,- docs_url=docs_url,- )- return task_id---async def _ingest_default_documents_url(- document_service,- models_service,- docs_url: str,- crawl_depth: int,- jwt_token=None,-):- """Ingest default docs from URL using OpenRAG ingestion logic (no Langflow)."""- if not docs_url:- raise ValueError("DEFAULT_DOCS_URL is not configured")-- logger.info(- "Running default URL docs ingestion with OpenRAG processor",- docs_url=docs_url,- crawl_depth=crawl_depth,- )- temp_file_path = await _materialize_default_docs_url_as_text_file(- docs_url=docs_url,- crawl_depth=crawl_depth,- )- try:- from models.processors import DocumentFileProcessor- from utils.hash_utils import hash_id-- from session_manager import AnonymousUser-- anonymous_user = AnonymousUser()-- processor = DocumentFileProcessor(- document_service,- models_service=models_service,- owner_user_id=None,- jwt_token=jwt_token,- owner_name=anonymous_user.name,- owner_email=anonymous_user.email,- is_sample_data=True,- connector_type="openrag_docs",- )- await processor.process_document_standard(- file_path=temp_file_path,- file_hash=hash_id(temp_file_path),- owner_user_id=None,- original_filename="openrag-url-default.txt",- jwt_token=jwt_token,- owner_name=anonymous_user.name,- owner_email=anonymous_user.email,- file_size=os.path.getsize(temp_file_path),- connector_type="openrag_docs",- is_sample_data=True,- )- finally:- try:- os.unlink(temp_file_path)- except FileNotFoundError:- pass- except Exception as e:- logger.error(- "Failed to clean temporary default URL docs file",- path=temp_file_path,- error=str(e),- )---async def _materialize_default_docs_url_as_text_file(- docs_url: str,- crawl_depth: int,-) -> str:- """Fetch URL content and write a temporary text file for OpenRAG ingestion."""- async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:- response = await client.get(docs_url)- response.raise_for_status()- raw_html = response.text-- title_match = re.search(- r"<title[^>]*>(.*?)</title\s*>",- raw_html,- flags=re.IGNORECASE | re.DOTALL,- )- title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"-- text_parser = _VisibleTextHTMLParser()- text_parser.feed(raw_html)- text_parser.close()- normalized_text = re.sub(r"\s+", " ", text_parser.get_text()).strip()-- content = (- f"{title}\n\n"- f"Source URL: {docs_url}\n"- f"Crawl depth: {crawl_depth}\n\n"- f"{normalized_text}\n"- )-- temp_file = tempfile.NamedTemporaryFile(- mode="w",- suffix=".txt",- prefix="openrag-url-default-",- delete=False,- encoding="utf-8",- )- with temp_file:- temp_file.write(content)- return temp_file.name---async def _delete_existing_default_docs(session_manager, connector_type: str):- """Delete previously ingested default OpenRAG docs before reingestion."""- from session_manager import AnonymousUser-- if session_manager is None:- logger.warning(- "Session manager unavailable; skipping default docs cleanup before reingestion"- )- return-- anonymous_user = AnonymousUser()- effective_jwt = None- if session_manager:- session_manager.get_user_opensearch_client(- anonymous_user.user_id, effective_jwt- )- if hasattr(session_manager, "_anonymous_jwt"):- effective_jwt = session_manager._anonymous_jwt-- opensearch_client = session_manager.get_user_opensearch_client(- anonymous_user.user_id, effective_jwt- )- delete_query = {- "query": {- "bool": {- "should": [- # URL-based default docs are ingested as system_default and- # owned by the anonymous onboarding user.- {- "bool": {- "must": [- {"term": {"connector_type": connector_type}},- {"term": {"owner_email": anonymous_user.email}},- ]- }- },- ],- "minimum_should_match": 1,- }- }- }- result = await opensearch_client.delete_by_query(- index=get_index_name(),- body=delete_query,- conflicts="proceed",- )- logger.info(- "Deleted existing default OpenRAG docs before reingestion",- deleted_chunks=result.get("deleted", 0),- )---async def _reingest_default_docs_on_upgrade_if_needed(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,-):- """Reingest default OpenRAG docs once when app version changes."""- config = get_openrag_config()-- previous_version = config.onboarding.openrag_docs_ingested_version- current_version = OPENRAG_VERSION- should_reingest = bool(previous_version) and previous_version != current_version-- # Legacy installs may not have a stored docs ingestion version.- # Use the presence of the OpenRAG docs filter as the signal that docs were- # already onboarded, independent of whether config.edited is set.- if not previous_version and config.onboarding.openrag_docs_filter_id:- should_reingest = True-- if not should_reingest:- return False-- logger.info(- "Detected OpenRAG upgrade; reingesting default docs",- previous_version=previous_version,- current_version=current_version,- )- await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")- await ingest_openrag_docs_when_ready(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- )- config.onboarding.openrag_docs_ingested_version = current_version- if _should_use_url_default_docs_ingest():- # Refresh signature metadata after upgrade reingestion so startup- # signature checks don't trigger an immediate duplicate ingest.- config.onboarding.openrag_docs_remote_signature = (- await _get_remote_docs_signature(DEFAULT_DOCS_URL)- )- else:- config.onboarding.openrag_docs_remote_signature = None- if not config_manager.save_config_file(config):- logger.warning(- "Default docs were reingested but failed to persist metadata",- current_version=current_version,- signature=config.onboarding.openrag_docs_remote_signature,- )- return True---async def _get_remote_docs_signature(docs_url: str):- """Get a signature for remote docs to detect content updates."""- try:- async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:- head_response = await client.head(docs_url)- if head_response.status_code >= 400:- get_response = await client.get(docs_url)- if get_response.status_code >= 400:- logger.warning(- "Failed to fetch remote docs signature",- docs_url=docs_url,- status_code=get_response.status_code,- )- return None- return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()-- etag = (head_response.headers.get("etag") or "").strip()- last_modified = (head_response.headers.get("last-modified") or "").strip()- if etag:- # Prefer ETag when available: it is typically the strongest- # cache validator and stays stable if extra cache headers- # appear/disappear without content changes.- return f"etag={etag}"- if last_modified:- return f"last_modified={last_modified}"-- # HEAD has no body. If cache headers are missing, fetch the page body.- get_response = await client.get(docs_url)- if get_response.status_code >= 400:- logger.warning(- "Failed to fetch remote docs signature body fallback",- docs_url=docs_url,- status_code=get_response.status_code,- )- return None- return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()- except Exception as e:- logger.error(- "Unable to retrieve remote docs signature",- docs_url=docs_url,- error=str(e),- )- return None---async def refresh_default_openrag_docs(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- force: bool = False,- reason: str = "startup",-):- """Refresh OpenRAG docs if remote content changed or when forced."""- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_START,- metadata={"reason": reason, "force": force},- )- try:- if not _should_use_url_default_docs_ingest():- logger.info(- "Skipping OpenRAG docs refresh: URL ingestion is not active",- ingest_source=DEFAULT_DOCS_INGEST_SOURCE,- disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,- has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),- has_docs_url=bool(DEFAULT_DOCS_URL),- )- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_SKIPPED,- metadata={- "reason": reason,- "force": force,- "skip_reason": "url_ingestion_inactive",- },- )- return False-- config = get_openrag_config()- if not config.edited:- logger.info("Skipping OpenRAG docs refresh: onboarding not completed")- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_SKIPPED,- metadata={- "reason": reason,- "force": force,- "skip_reason": "onboarding_not_completed",- },- )- return False-- signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)- if not signature and not force:- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_SKIPPED,- metadata={- "reason": reason,- "force": force,- "skip_reason": "signature_unavailable",- },- )- return False-- previous_signature = config.onboarding.openrag_docs_remote_signature- should_refresh = force or (- signature is not None and signature != previous_signature- )- if not should_refresh:- logger.info(- "OpenRAG docs refresh skipped: remote signature unchanged",- signature=signature,- )- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_SKIPPED,- metadata={- "reason": reason,- "force": force,- "skip_reason": "signature_unchanged",- },- )- return False-- logger.info(- "Refreshing default OpenRAG docs",- reason=reason,- force=force,- previous_signature=previous_signature,- new_signature=signature,- )- await _delete_existing_default_docs(- session_manager, connector_type="openrag_docs"- )- await ingest_openrag_docs_when_ready(- document_service,- models_service,- task_service,- langflow_file_service,- session_manager,- )- config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION- # Keep docs version/signature metadata consistent after a refresh.- # If signature retrieval failed, persist None explicitly instead of- # leaving a stale previous signature value.- config.onboarding.openrag_docs_remote_signature = signature- if not config_manager.save_config_file(config):- logger.warning(- "OpenRAG docs refreshed but failed to persist metadata",- version=config.onboarding.openrag_docs_ingested_version,- signature=config.onboarding.openrag_docs_remote_signature,- )- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_COMPLETE,- metadata={"reason": reason, "force": force},- )- return True- except Exception as e:- await TelemetryClient.send_event(- Category.DOCUMENT_INGESTION,- MessageId.ORB_DOC_REFRESH_FAILED,- metadata={- "reason": reason,- "force": force,- "error_type": type(e).__name__,- },- )- raise---async def health_check(request: Request):- """Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""- return JSONResponse({"status": "ok"}, status_code=200)---async def opensearch_health_ready(request):- """Readiness probe: verifies OpenSearch dependency is reachable."""- from config.settings import IBM_AUTH_ENABLED, OPENSEARCH_URL-- if IBM_AUTH_ENABLED:- logger.debug(- "[OPENSEARCH] OpenSearch auth mode enabled, health check per-request"- )- # In IBM auth mode we cannot rely on the global OpenSearch client- # (auth is established per-request), so perform a lightweight,- # unauthenticated connectivity check against the OpenSearch endpoint.- opensearch_url = OPENSEARCH_URL.rstrip("/")- try:- timeout = httpx.Timeout(5.0)- async with httpx.AsyncClient(timeout=timeout) as client:- resp = await client.get(f"{opensearch_url}/")- if resp.status_code < 500:- logger.debug("[OPENSEARCH] OpenSearch health check successful")- return JSONResponse(- {- "status": "ready",- "dependencies": {"opensearch": "up"},- "note": "OpenSearch auth mode - connectivity verified via unauthenticated probe",- },- status_code=200,- )- else:- logger.debug("[OPENSEARCH] OpenSearch health check failed")- return JSONResponse(- {- "status": "not_ready",- "dependencies": {"opensearch": "down"},- "error": f"Unexpected status from OpenSearch: {resp.status_code}",- },- status_code=503,- )- except Exception as e:- logger.error(- "[OPENSEARCH] OpenSearch health check failed", error=str(e)- )- return JSONResponse(- {- "status": "not_ready",- "dependencies": {"opensearch": "down"},- "error": "OpenSearch health check failed",- },- status_code=503,- )-- try:- await asyncio.wait_for(clients.opensearch.info(), timeout=5.0)- return JSONResponse(- {"status": "ready", "dependencies": {"opensearch": "up"}},- status_code=200,- )- except Exception as e:- logger.error(- "[OPENSEARCH] OpenSearch health check failed", error=str(e)- )- return JSONResponse(- {- "status": "not_ready",- "dependencies": {"opensearch": "down"},- "error": "OpenSearch health check failed",- },- status_code=503,- )---async def _ingest_default_documents_openrag(- document_service,- models_service,- task_service,- file_paths,- connector_type: str = "openrag_docs",- existing_task_id: str = None,- jwt_token=None,-):- """Ingest default documents using traditional OpenRAG processor."""- logger.info(- "Using traditional OpenRAG ingestion for default documents",- file_count=len(file_paths),- )-- from models.processors import DocumentFileProcessor-- from session_manager import AnonymousUser- anonymous_user = AnonymousUser()-- processor = DocumentFileProcessor(- document_service,- models_service=models_service,- owner_user_id=None,- jwt_token=jwt_token,- owner_name=anonymous_user.name,- owner_email=anonymous_user.email,- is_sample_data=True,- connector_type=connector_type,- )-- task_id = await task_service.create_custom_task(- "anonymous", file_paths, processor, existing_task_id=existing_task_id- )- logger.info(- "Started traditional OpenRAG ingestion task",- task_id=task_id,- file_count=len(file_paths),- )- return task_id--async def _update_mcp_server_urls(langflow_mcp_service):- """Update MCP server URLs (patch localhost and convert to streamable HTTP)."""- try:- result = await langflow_mcp_service.update_all_mcp_server_urls()- logger.info("Updated MCP server URLs after settings change", **result)- except Exception as mcp_error:- logger.warning(f"Failed to update MCP server URLs after settings change: {str(mcp_error)}")---async def startup_tasks(services):- """Startup tasks"""- from config.settings import IBM_AUTH_ENABLED-- logger.info("Starting startup tasks")- await TelemetryClient.send_event(- Category.APPLICATION_STARTUP, MessageId.ORB_APP_START_INIT- )-- # Update model registry to allow further search calls to be instant- try:- models_service = services["models_service"]- await models_service.update_model_registry()- except Exception as e:- logger.error(- "Failed to update model registry at startup — "- "models may be missing until the next restart",- error=str(e),- )---- if IBM_AUTH_ENABLED:- logger.info(- "IBM auth mode: skipping startup OpenSearch checks. "- "OpenSearch will be initialized during onboarding with user credentials."- )- else:- # Only initialize basic OpenSearch connection, not the index- # Index will be created after onboarding when we know the embedding model- await wait_for_opensearch()-- # Setup OpenSearch security (roles and mappings) after connection is established- try:- from utils.opensearch_utils import setup_opensearch_security- await setup_opensearch_security(clients.opensearch)- logger.info("OpenSearch security configuration completed successfully")- except Exception as e:- logger.warning(- "Failed to setup OpenSearch security configuration - continuing anyway",- error=str(e)- )-- if DISABLE_INGEST_WITH_LANGFLOW:- await _ensure_opensearch_index()-- # Ensure that the OpenSearch index exists if onboarding was already completed- # - Handles the case where OpenSearch is reset (e.g., volume deleted) after onboarding- embedding_model = None- try:- config = get_openrag_config()- embedding_model = config.knowledge.embedding_model-- if config.edited and embedding_model:- logger.info(- "Ensuring that the OpenSearch index exists (after onboarding)...",- embedding_model=embedding_model,- )-- await init_index()-- logger.info(- "Successfully ensured that the OpenSearch index exists (after onboarding).",- embedding_model=embedding_model,- )- except Exception as e:- logger.error(- "Failed to ensure that the OpenSearch index exists (after onboarding).",- embedding_model=embedding_model,- error=str(e),- )- raise-- # Configure alerting security- await configure_alerting_security()-- # Reingest bundled OpenRAG docs once after application upgrade.- upgrade_reingested = False- try:- upgrade_reingested = await _reingest_default_docs_on_upgrade_if_needed(- services["document_service"],- services["models_service"],- services["task_service"],- services["langflow_file_service"],- services["session_manager"],- )- except Exception as e:- logger.error("Default docs reingestion on upgrade failed", error=str(e))-- if FETCH_OPENRAG_DOCS_AT_STARTUP and not upgrade_reingested:- try:- await refresh_default_openrag_docs(- services["document_service"],- services["models_service"],- services["task_service"],- services["langflow_file_service"],- services["session_manager"],- force=False,- reason="startup",- )- except Exception as e:- logger.error("OpenRAG docs startup refresh failed", error=str(e))-- # Update MCP server URLs (patch localhost and convert to streamable HTTP)- await _update_mcp_server_urls(services["langflow_mcp_service"])-- # Ensure all configured flows exist in Langflow (create-only, never overwrites).- # This replaces LANGFLOW_LOAD_FLOWS_PATH, which performed a blind upsert on- # every container start and discarded any user edits made in the Langflow UI.- newly_created: set[str] = set()- try:- flows_service = services["flows_service"]- newly_created = await flows_service.ensure_flows_exist()- except Exception as e:- logger.error(- "Failed to ensure Langflow flows exist at startup — "- "flows may be missing until the next restart",- error=str(e),- )-- # Check if flows were reset and reapply settings if config is edited- try:- config = get_openrag_config()- if config.edited:- logger.info("Checking if Langflow flows were reset")- flows_service = services["flows_service"]- reset_flows = await flows_service.check_flows_reset()- # Exclude flows that were just seeded — they match the JSON by design,- # not because they were externally reset.- reset_flows = [f for f in reset_flows if f not in newly_created]-- if reset_flows:- logger.info(- f"Detected reset flows: {', '.join(reset_flows)}. Reapplying all settings."- )- await TelemetryClient.send_event(- Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_RESET_DETECTED- )- from api.settings import reapply_all_settings-- await reapply_all_settings(session_manager=services["session_manager"])- logger.info(- "Successfully reapplied settings after detecting flow resets"- )- await TelemetryClient.send_event(- Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_SETTINGS_REAPPLIED- )- else:- logger.info(- "No flows detected as reset, skipping settings reapplication"- )- else:- logger.debug("Configuration not yet edited, skipping flow reset check")- except Exception as e:- logger.error(f"Failed to check flows reset or reapply settings: {str(e)}")- await TelemetryClient.send_event(- Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_RESET_CHECK_FAIL- )- # Don't fail startup if this check fails---async def initialize_services():- """Initialize all services and their dependencies"""- await TelemetryClient.send_event(- Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_START- )- from config.settings import IBM_AUTH_ENABLED-- if IBM_AUTH_ENABLED:- logger.info("IBM auth mode enabled — JWT validation delegated to Traefik")-- # Generate JWT keys if they don't exist, a JWT signing key isn't specified,- # and IBM auth is not enabled (IBM mode delegates all auth to Traefik)- if not os.getenv("JWT_SIGNING_KEY") and not IBM_AUTH_ENABLED:- generate_jwt_keys()-- # Initialize clients (now async to generate Langflow API key)- try:- await clients.initialize()- except Exception as e:- logger.error("Failed to initialize clients", error=str(e))- await TelemetryClient.send_event(- Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_OS_CLIENT_FAIL- )- raise-- # Initialize session manager- session_manager = SessionManager(SESSION_SECRET)-- # Initialize services- models_service = ModelsService()- document_service = DocumentService(- session_manager=session_manager,- models_service=models_service,- docling_service=clients.docling_service,- )- search_service = SearchService(session_manager, models_service)- register_search_service(search_service)- task_service = TaskService(- document_service,- models_service,- ingestion_timeout=INGESTION_TIMEOUT,- docling_service=clients.docling_service,- )- flows_service = FlowsService()- chat_service = ChatService(flows_service=flows_service)- knowledge_filter_service = KnowledgeFilterService(session_manager)- monitor_service = MonitorService(session_manager)- langflow_file_service = LangflowFileService(- flows_service=flows_service,- docling_service=clients.docling_service,- )- langflow_mcp_service = LangflowMCPService()-- # Initialize both connector services- langflow_connector_service = LangflowConnectorService(- task_service=task_service,- session_manager=session_manager,- flows_service=flows_service,- docling_service=clients.docling_service,- )- openrag_connector_service = ConnectorService(- patched_async_client=clients,- embed_model=get_embedding_model(),- index_name=get_index_name(),- task_service=task_service,- session_manager=session_manager,- models_service=models_service,- document_service=document_service,- docling_service=clients.docling_service,- )-- # Create connector router that chooses based on configuration- connector_service = ConnectorRouter(- langflow_connector_service=langflow_connector_service,- openrag_connector_service=openrag_connector_service,- )-- # Initialize auth service- auth_service = AuthService(- session_manager,- connector_service,- flows_service,- langflow_mcp_service=langflow_mcp_service,- )-- # Load persisted connector connections at startup so webhooks and syncs- # can resolve existing subscriptions immediately after server boot- # Skip in no-auth mode since connectors require OAuth-- if not is_no_auth_mode():- try:- await connector_service.initialize()- loaded_count = len(connector_service.connection_manager.connections)- logger.info(- "Loaded persisted connector connections on startup",- loaded_count=loaded_count,- )- except Exception as e:- logger.error(- "Failed to load persisted connections on startup", error=str(e)- )- await TelemetryClient.send_event(- Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_LOAD_FAILED- )- else:- logger.info("[CONNECTORS] Skipping connection loading in no-auth mode")-- await TelemetryClient.send_event(- Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_SUCCESS- )-- # API Key service for public API authentication- api_key_service = APIKeyService(session_manager)-- # ===== RBAC service =====- # We do NOT open the SQL engine here. `create_app()` runs inside an- # `asyncio.run(...)` whose loop is closed BEFORE uvicorn starts its- # own loop — any AsyncEngine bound to this loop would be dead by- # then. Instead, RBACService takes a *lazy* session-factory getter- # that resolves `db.engine.SessionLocal` at call time — that- # attribute is filled in by the lifespan startup event running on- # uvicorn's loop. Alembic upgrade is run synchronously from __main__- # before `asyncio.run(create_app())` so the schema is in place by- # the time the lifespan opens the engine.- from db import engine as _db_engine_mod- from services.rbac_service import RBACService- from services.workspace_config_service import WorkspaceConfigService-- def _lazy_session_factory():- sl = _db_engine_mod.SessionLocal- if sl is None:- raise RuntimeError(- "DB engine not yet initialized. RBACService called before lifespan startup."- )- return sl()-- rbac_service = RBACService(_lazy_session_factory)-- # WorkspaceConfigService — DB-first reads of what config.yaml holds,- # with the legacy ConfigManager kept as the yaml fallback during- # Phase B (dual-write).- workspace_config_service = WorkspaceConfigService(- config_manager=config_manager,- session_factory=_lazy_session_factory,- )-- # Plumb the session factory into the two chat-history services- # (session_ownership + conversation_persistence). They lazy-resolve- # `db.engine.SessionLocal` as a fallback, but setting it here makes- # the wiring explicit and avoids the import path on the hot loop.- from services.session_ownership_service import session_ownership_service- from services.conversation_persistence_service import conversation_persistence- session_ownership_service._session_factory = _lazy_session_factory- conversation_persistence._session_factory = _lazy_session_factory-- return {- "document_service": document_service,- "search_service": search_service,- "task_service": task_service,- "chat_service": chat_service,- "flows_service": flows_service,- "langflow_file_service": langflow_file_service,- "auth_service": auth_service,- "connector_service": connector_service,- "knowledge_filter_service": knowledge_filter_service,- "models_service": models_service,- "monitor_service": monitor_service,- "session_manager": session_manager,- "api_key_service": api_key_service,- "langflow_mcp_service": langflow_mcp_service,- "docling_service": clients.docling_service,- "rbac_service": rbac_service,- "workspace_config_service": workspace_config_service,- }---async def _periodic_backup(services):- """Periodic backup task that runs every 5 minutes"""- while True:- try:- await asyncio.sleep(5 * 60) # Wait 5 minutes-- # Check if onboarding has been completed- config = get_openrag_config()- if not config.edited:- logger.info("[CONFIG] Onboarding not completed yet, skipping periodic backup")- continue-- flows_service = services.get("flows_service")- if flows_service:- logger.info("Running periodic flow backup")- backup_results = await flows_service.backup_all_flows(only_if_changed=True)- if backup_results["backed_up"]:- logger.info(- "Periodic backup completed",- backed_up=len(backup_results["backed_up"]),- skipped=len(backup_results["skipped"]),- )- else:- logger.debug(- "Periodic backup: no flows changed",- skipped=len(backup_results["skipped"]),- )- except asyncio.CancelledError:- logger.info("Periodic backup task cancelled")- break- except Exception as e:- logger.error(f"Error in periodic backup task: {str(e)}")---@asynccontextmanager-async def lifespan(app: FastAPI):- """FastAPI lifespan handler for startup and shutdown tasks"""- # STARTUP-- services = app.state.services-- # 1. Start MCP lifespan (must happen first)- mcp_lifespan_ctx = getattr(app.state, "mcp_lifespan_ctx", None)- if mcp_lifespan_ctx:- await mcp_lifespan_ctx.__aenter__()- logger.info("FastMCP lifespan started")-- # 2. Telemetry- await TelemetryClient.send_event(Category.APPLICATION_STARTUP, MessageId.ORB_APP_STARTED)-- # 3. Startup tasks (background)- t1 = asyncio.create_task(startup_tasks(services))- app.state.background_tasks.add(t1)- t1.add_done_callback(app.state.background_tasks.discard)-- # 4. Periodic tasks- services["task_service"].start_cleanup_scheduler()- backup_task = asyncio.create_task(_periodic_backup(services))- app.state.background_tasks.add(backup_task)- backup_task.add_done_callback(app.state.background_tasks.discard)-- yield-- # SHUTDOWN- logger.info("Application shutdown span initiated")-- # 1. Telemetry shutdown event- await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)-- # 2. Stop MCP lifespan- if mcp_lifespan_ctx:- await mcp_lifespan_ctx.__aexit__(None, None, None)- logger.info("FastMCP lifespan stopped")-- # 3. Graceful OpenSearch shutdown- try:- from utils.opensearch_utils import graceful_opensearch_shutdown-- await graceful_opensearch_shutdown(clients.opensearch)- except Exception as e:- logger.error("Error during graceful OpenSearch shutdown", error=str(e))-- # 4. Subscriptions cleanup- await cleanup_subscriptions_proper(services)-- # 5. Task service shutdown- await services["task_service"].shutdown()-- # 6. Global clients cleanup- await clients.cleanup()-- # 6b. SQL engine cleanup- try:- from db.engine import dispose_engine- await dispose_engine()- except Exception as e:- logger.error("Error disposing DB engine", error=str(e))-- # 7. Telemetry client cleanup- from utils.telemetry.client import cleanup_telemetry_client-- await cleanup_telemetry_client()-- logger.info("Application shutdown span completed")----class RequestLoggingMiddleware:- """Pure ASGI middleware that assigns a correlation ID to every request and- emits a single structured [API] log line with method, path, status code,- and duration after the response is sent.-- Using pure ASGI (not BaseHTTPMiddleware) so structlog contextvars bound- inside endpoint handlers propagate back correctly to this middleware.- """-- def __init__(self, app: ASGIApp) -> None:- self.app = app-- async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:- if scope["type"] != "http":- await self.app(scope, receive, send)- return-- import structlog as _structlog-- _structlog.contextvars.clear_contextvars()-- headers_dict = dict(scope.get("headers", []))- request_id = (- headers_dict.get(b"x-request-id", b"").decode() or str(uuid.uuid4())- )- path = scope.get("path", "")- method = scope.get("method", "")-- _structlog.contextvars.bind_contextvars(- request_id=request_id,- method=method,- path=path,- )-- status_code = 500- start = time.perf_counter()-- async def send_wrapper(message):- nonlocal status_code- if message["type"] == "http.response.start":- status_code = message["status"]- # Inject correlation ID into response headers- headers = list(message.get("headers", []))- headers.append((b"x-request-id", request_id.encode()))- message = {**message, "headers": headers}- await send(message)-- try:- await self.app(scope, receive, send_wrapper)- finally:- duration_ms = round((time.perf_counter() - start) * 1000)- level = "warning" if status_code >= 500 else "info"- getattr(logger, level)(- "[API] Request",- status_code=status_code,- duration_ms=duration_ms,- )- _structlog.contextvars.clear_contextvars()---async def create_app():- """Create and configure the FastAPI application"""- services = await initialize_services()-- app = FastAPI(title="OpenRAG API", version=OPENRAG_VERSION, debug=True)- app.state.services = services # Store services for cleanup- app.state.background_tasks = set()-- # Wire up ASGI request logging middleware (pure ASGI, not BaseHTTPMiddleware)- app.add_middleware(RequestLoggingMiddleware)-- try:- Instrumentator().instrument(app).expose(app)- except Exception as e:- logger.error(f"Failed to instrument app with Prometheus: {str(e)}")-- # Register route handlers — auth and service injection done via FastAPI Depends() in each handler-- # Langflow Files endpoints- app.add_api_route(- "/langflow/files/upload",- langflow_files.upload_user_file,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/langflow/ingest",- langflow_files.run_ingestion,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/langflow/files",- langflow_files.delete_user_files,- methods=["DELETE"],- tags=["internal"],- )- app.add_api_route(- "/langflow/upload_ingest",- langflow_files.upload_and_ingest_user_file,- methods=["POST"],- tags=["internal"],- )-- # Upload endpoints- app.add_api_route(- "/upload_context", upload.upload_context, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/upload_path", upload.upload_path, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/upload_options", upload.upload_options, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/upload_bucket", upload.upload_bucket, methods=["POST"], tags=["internal"]- )-- # Task endpoints- app.add_api_route(- "/tasks/{task_id}", tasks.task_status, methods=["GET"], tags=["internal"]- )- app.add_api_route("/tasks", tasks.all_tasks, methods=["GET"], tags=["internal"])- app.add_api_route(- "/tasks/{task_id}/cancel",- tasks.cancel_task,- methods=["POST"],- tags=["internal"],- )-- # Search endpoint- app.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])-- # Knowledge Filter endpoints- app.add_api_route(- "/knowledge-filter",- knowledge_filter.create_knowledge_filter,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/search",- knowledge_filter.search_knowledge_filters,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/{filter_id}",- knowledge_filter.get_knowledge_filter,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/{filter_id}",- knowledge_filter.update_knowledge_filter,- methods=["PUT"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/{filter_id}",- knowledge_filter.delete_knowledge_filter,- methods=["DELETE"],- tags=["internal"],- )-- # Knowledge Filter Subscription endpoints- app.add_api_route(- "/knowledge-filter/{filter_id}/subscribe",- knowledge_filter.subscribe_to_knowledge_filter,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/{filter_id}/subscriptions",- knowledge_filter.list_knowledge_filter_subscriptions,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/knowledge-filter/{filter_id}/subscribe/{subscription_id}",- knowledge_filter.cancel_knowledge_filter_subscription,- methods=["DELETE"],- tags=["internal"],- )-- # Knowledge Filter Webhook endpoint (no auth required - called by OpenSearch)- app.add_api_route(- "/knowledge-filter/{filter_id}/webhook/{subscription_id}",- knowledge_filter.knowledge_filter_webhook,- methods=["POST"],- tags=["internal"],- )-- # Chat endpoints- app.add_api_route("/chat", chat.chat_endpoint, methods=["POST"], tags=["internal"])- app.add_api_route(- "/langflow", chat.langflow_endpoint, methods=["POST"], tags=["internal"]- )-- # Chat history endpoints- app.add_api_route(- "/chat/history", chat.chat_history_endpoint, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/langflow/history",- chat.langflow_history_endpoint,- methods=["GET"],- tags=["internal"],- )-- # Session deletion endpoint- app.add_api_route(- "/sessions/{session_id}",- chat.delete_session_endpoint,- methods=["DELETE"],- tags=["internal"],- )-- # Authentication endpoints- app.add_api_route("/auth/init", auth.auth_init, methods=["POST"], tags=["internal"])- app.add_api_route(- "/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"]- )- app.add_api_route("/auth/me", auth.auth_me, methods=["GET"], tags=["internal"])- app.add_api_route(- "/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/auth/ibm/login", auth.ibm_login, methods=["POST"], tags=["internal"]- )-- # Connector endpoints- app.add_api_route(- "/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"]- )- # IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)- app.add_api_route(- "/connectors/ibm_cos/defaults",- ibm_cos_defaults,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/ibm_cos/configure",- ibm_cos_configure,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/ibm_cos/{connection_id}/buckets",- ibm_cos_list_buckets,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/ibm_cos/{connection_id}/bucket-status",- ibm_cos_bucket_status,- methods=["GET"],- tags=["internal"],- )- # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)- app.add_api_route(- "/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/connectors/aws_s3/configure",- s3_configure,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/aws_s3/{connection_id}/buckets",- s3_list_buckets,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/aws_s3/{connection_id}/bucket-status",- s3_bucket_status,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/sync",- connectors.connector_sync,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/sync-all",- connectors.sync_all_connectors,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/status",- connectors.connector_status,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/token",- connectors.connector_token,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/disconnect",- connectors.connector_disconnect,- methods=["DELETE"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/webhook",- connectors.connector_webhook,- methods=["POST", "GET"],- tags=["internal"],- )-- # Document endpoints- app.add_api_route(- "/documents/check-filename",- documents.check_filename_exists,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/documents/delete-by-filename",- documents.delete_documents_by_filename,- methods=["POST"],- tags=["internal"],- )-- # OIDC endpoints- app.add_api_route(- "/.well-known/openid-configuration",- oidc.oidc_discovery,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/auth/introspect",- oidc.token_introspection,- methods=["POST"],- tags=["internal"],- )-- # Settings endpoints- app.add_api_route(- "/settings", settings.get_settings, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/settings", settings.update_settings, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/onboarding/state",- settings.update_onboarding_state,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/openrag-docs/refresh",- settings.refresh_openrag_docs,- methods=["POST"],- tags=["internal"],- )-- # Provider health check endpoint- app.add_api_route(- "/provider/health",- provider_health.check_provider_health,- methods=["GET"],- tags=["internal"],- )-- # Health check endpoints- app.add_api_route("/health", health_check, methods=["GET"], tags=["internal"])- app.add_api_route(- "/search/health", opensearch_health_ready, methods=["GET"], tags=["internal"]- )-- # Models endpoints- app.add_api_route(- "/models/openai", models.get_openai_models, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/models/anthropic",- models.get_anthropic_models,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/models/ollama", models.get_ollama_models, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/models/ibm", models.get_ibm_models, methods=["POST"], tags=["internal"]- )-- # Onboarding endpoints- app.add_api_route(- "/onboarding", settings.onboarding, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/onboarding/rollback",- settings.rollback_onboarding,- methods=["POST"],- tags=["internal"],- )-- # Docling preset update endpoint- app.add_api_route(- "/settings/docling-preset",- settings.update_docling_preset,- methods=["PATCH"],- tags=["internal"],- )-- # Nudges endpoints- app.add_api_route(- "/nudges", nudges.nudges_from_kb_endpoint, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/nudges/{chat_id}",- nudges.nudges_from_chat_id_endpoint,- methods=["POST"],- tags=["internal"],- )-- # Flow reset endpoint- app.add_api_route(- "/reset-flow/{flow_type}",- flows.reset_flow_endpoint,- methods=["POST"],- tags=["internal"],- )-- # Router upload ingest endpoint- app.add_api_route(- "/router/upload_ingest",- router.upload_ingest_router,- methods=["POST"],- tags=["internal"],- )-- # Docling service proxy- app.add_api_route(- "/docling/health", docling.health, methods=["GET"], tags=["internal"]- )-- # ===== Users / RBAC Endpoints (JWT auth) =====- from api import users as users_api- from api.admin import rbac as admin_rbac- from api import config as config_api- app.include_router(users_api.router)- app.include_router(admin_rbac.router)- # Public — must work pre-auth so the onboarding wizard can render.- app.include_router(config_api.router)-- # ===== API Key Management Endpoints (JWT auth for UI) =====- app.add_api_route(- "/keys", api_keys.list_keys_endpoint, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/keys", api_keys.create_key_endpoint, methods=["POST"], tags=["internal"]- )- app.add_api_route(- "/keys/{key_id}",- api_keys.revoke_key_endpoint,- methods=["DELETE"],- tags=["internal"],- )-- # ===== Public API v1 Endpoints (API Key auth) =====- # Chat endpoints- app.add_api_route(- "/v1/chat", v1_chat.chat_create_endpoint, methods=["POST"], tags=["public"]- )- app.add_api_route(- "/v1/chat", v1_chat.chat_list_endpoint, methods=["GET"], tags=["public"]- )- app.add_api_route(- "/v1/chat/{chat_id}",- v1_chat.chat_get_endpoint,- methods=["GET"],- tags=["public"],- )- app.add_api_route(- "/v1/chat/{chat_id}",- v1_chat.chat_delete_endpoint,- methods=["DELETE"],- tags=["public"],- )-- # Search endpoint- app.add_api_route(- "/v1/search", v1_search.search_endpoint, methods=["POST"], tags=["public"]- )-- # Documents endpoints- app.add_api_route(- "/v1/documents/ingest",- v1_documents.ingest_endpoint,- methods=["POST"],- tags=["public"],- )- app.add_api_route(- "/v1/tasks/{task_id}",- v1_documents.task_status_endpoint,- methods=["GET"],- tags=["public"],- )- app.add_api_route(- "/v1/documents",- v1_documents.delete_document_endpoint,- methods=["DELETE"],- tags=["public"],- )-- # Settings endpoints- app.add_api_route(- "/v1/settings",- v1_settings.get_settings_endpoint,- methods=["GET"],- tags=["public"],- )- app.add_api_route(- "/v1/settings",- v1_settings.update_settings_endpoint,- methods=["POST"],- tags=["public"],- )-- # Models endpoint- app.add_api_route(- "/v1/models/{provider}",- v1_models.list_models_endpoint,- methods=["GET"],- tags=["public"],- )-- # Knowledge filters endpoints- app.add_api_route(- "/v1/knowledge-filters",- v1_knowledge_filters.create_endpoint,- methods=["POST"],- tags=["public"],- )- app.add_api_route(- "/v1/knowledge-filters/search",- v1_knowledge_filters.search_endpoint,- methods=["POST"],- tags=["public"],- )- app.add_api_route(- "/v1/knowledge-filters/{filter_id}",- v1_knowledge_filters.get_endpoint,- methods=["GET"],- tags=["public"],- )- app.add_api_route(- "/v1/knowledge-filters/{filter_id}",- v1_knowledge_filters.update_endpoint,- methods=["PUT"],- tags=["public"],- )- app.add_api_route(- "/v1/knowledge-filters/{filter_id}",- v1_knowledge_filters.delete_endpoint,- methods=["DELETE"],- tags=["public"],- )-- # ===== FastMCP Streamable HTTP Server =====- # Exposes /v1/* endpoints as MCP tools at POST /mcp- # Client config: { "url": "http://localhost:8000/mcp", "headers": { "X-API-Key": "..." } }- logger.info("Creating MCP server")- mcp_server = create_mcp_server(app)- mcp_http_app = mcp_server.http_app(transport="streamable-http", path="/")- app.mount("/mcp", mcp_http_app)- logger.info("MCP server mounted at /mcp (streamable-http)")-- # FastMCP requires its own lifespan to be run so that the- # StreamableHTTPSessionManager task group is initialized before requests arrive.- # FastAPI does not automatically propagate lifespan to mounted sub-apps,- # so we wire it in manually via startup/shutdown handlers.- _mcp_lifespan_ctx = mcp_http_app.router.lifespan_context(mcp_http_app)-- async def _start_mcp_lifespan():- await _mcp_lifespan_ctx.__aenter__()- logger.info("FastMCP lifespan started")-- async def _stop_mcp_lifespan():- await _mcp_lifespan_ctx.__aexit__(None, None, None)- logger.info("FastMCP lifespan stopped")-- app.add_event_handler("startup", _start_mcp_lifespan)- app.add_event_handler("shutdown", _stop_mcp_lifespan)-- # Add startup event handler- @app.on_event("startup")- async def startup_event():- # Hard-fail if the operator has configured multiple workers. The- # RBAC permission cache and the OAuth-subject→DB-id cache are- # both per-process; a second worker silently sees stale- # permissions for up to OPENRAG_PERM_CACHE_TTL seconds after- # role mutations. Until the cache moves to a shared backend- # (Redis), this constraint is real and must be enforced.- _workers = int(os.getenv("UVICORN_WORKERS", "1") or "1")- if _workers > 1:- logger.error(- "Multi-worker deployment unsupported until cache is "- "shared across processes. Set UVICORN_WORKERS=1.",- requested_workers=_workers,- )- raise RuntimeError("UVICORN_WORKERS>1 is not supported")- cache_backend = os.getenv("CACHE_BACKEND", "memory").lower()- if cache_backend not in ("memory",):- logger.error(- "Unsupported CACHE_BACKEND. Only 'memory' is wired.",- requested=cache_backend,- )- raise RuntimeError(f"unsupported CACHE_BACKEND={cache_backend!r}")- logger.info(- "Permission cache configured",- backend=cache_backend,- workers=_workers,- perm_cache_ttl_s=int(os.getenv("OPENRAG_PERM_CACHE_TTL", "60") or "60"),- )-- # RBAC kill-switch visibility. OPENRAG_RBAC_ENFORCE=false makes- # every authenticated user effectively admin — log loudly so- # operators see it on every boot. Available in all run modes;- # the operator owns the trade-off.- from services.rbac_service import is_rbac_enforced- from utils.run_mode_utils import get_run_mode- if is_rbac_enforced():- logger.info("RBAC enforcement is ON", run_mode=get_run_mode())- else:- logger.warning(- "RBAC enforcement is DISABLED — every authenticated "- "user has full access via the OPENRAG_RBAC_ENFORCE=false "- "kill switch.",- run_mode=get_run_mode(),- )-- # Open the SQL engine on uvicorn's live loop (NOT the one used by- # asyncio.run(create_app()) which closed already). All RBAC code- # uses RBACService's lazy factory that reads db.engine.SessionLocal- # at call time, so it will pick up the binding we set here.- try:- from db.engine import init_engine- init_engine()- except Exception as e:- logger.error("DB engine init failed at startup", error=str(e))- raise-- # One-shot JSON->DB migration + test-fixture cleanup. Runs once- # per install, idempotent via migration_status rows.- try:- from db.engine import SessionLocal as _SL- from db.migrations_runtime import run as run_runtime_migration- if _SL is not None:- async with _SL() as _session:- await run_runtime_migration(_session)- await _session.commit()- except Exception as e:- logger.error("Runtime DB migration failed", error=str(e))- raise-- await TelemetryClient.send_event(- Category.APPLICATION_STARTUP, MessageId.ORB_APP_STARTED- )- # Start index initialization in background to avoid blocking OIDC endpoints- t1 = asyncio.create_task(startup_tasks(services))- app.state.background_tasks.add(t1)- t1.add_done_callback(app.state.background_tasks.discard)-- # Start periodic task cleanup scheduler- services["task_service"].start_cleanup_scheduler()-- # Start periodic flow backup task (every 5 minutes)- async def periodic_backup():- """Periodic backup task that runs every 15 minutes"""- while True:- try:- await asyncio.sleep(5 * 60) # Wait 5 minutes-- # Check if onboarding has been completed- config = get_openrag_config()- if not config.edited:- logger.debug(- "Onboarding not completed yet, skipping periodic backup"- )- continue-- flows_service = services.get("flows_service")- if flows_service:- logger.info("Running periodic flow backup")- backup_results = await flows_service.backup_all_flows(- only_if_changed=True- )- if backup_results["backed_up"]:- logger.info(- "Periodic backup completed",- backed_up=len(backup_results["backed_up"]),- skipped=len(backup_results["skipped"]),- )- else:- logger.debug(- "Periodic backup: no flows changed",- skipped=len(backup_results["skipped"]),- )- except asyncio.CancelledError:- logger.info("Periodic backup task cancelled")- break- except Exception as e:- logger.error(f"Error in periodic backup task: {str(e)}")- # Continue running even if one backup fails-- backup_task = asyncio.create_task(periodic_backup())- app.state.background_tasks.add(backup_task)- backup_task.add_done_callback(app.state.background_tasks.discard)-- # Add shutdown event handler- @app.on_event("shutdown")- async def shutdown_event():- await TelemetryClient.send_event(- Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN- )- logger.info("Application shutdown initiated")-- # Drain any pending workspace_config DB-mirror tasks before we- # tear down the engine. Without this, a save_config triggered- # right before shutdown can be cancelled mid-write, leaving- # yaml and DB out of sync.- try:- wcs = services.get("workspace_config_service") if isinstance(services, dict) else None- if wcs is not None:- await wcs.await_pending_mirrors()- except Exception as e: # noqa: BLE001- logger.error("Error awaiting pending DB mirrors", error=str(e))-- # Gracefully shutdown OpenSearch connection first- try:- from utils.opensearch_utils import graceful_opensearch_shutdown- await graceful_opensearch_shutdown(clients.opensearch)- except Exception as e:- logger.error("Error during graceful OpenSearch shutdown", error=str(e))-- await cleanup_subscriptions_proper(services)- # Cleanup task service (cancels background tasks and process pool)- await services["task_service"].shutdown()- # Cleanup async clients (this will also close OpenSearch client if not already closed)- await clients.cleanup()- # Cleanup telemetry client- from utils.telemetry.client import cleanup_telemetry_client-- await cleanup_telemetry_client()- logger.info("Application shutdown completed")-- return app+__all__ = [+ "create_app",+ "startup_tasks",+ "generate_jwt_keys",+ "init_index",+ "_ensure_opensearch_index",+ "ingest_default_documents_when_ready",+ "refresh_default_openrag_docs",+ "_get_remote_docs_signature",+ "_should_use_url_default_docs_ingest",+]def cleanup():- """Cleanup on application shutdown"""- # Cleanup process pools only (webhooks handled by FastAPI shutdown)+ """Cleanup on application shutdown (atexit hook)."""logger.info("Application shutting down")- pass---async def cleanup_subscriptions_proper(services):- """Cancel all active webhook subscriptions"""- logger.info("Cancelling active webhook subscriptions")-- try:- connector_service = services["connector_service"]- await connector_service.connection_manager.load_connections()-- # Get all active connections with webhook subscriptions- all_connections = await connector_service.connection_manager.list_connections()- active_connections = [- c- for c in all_connections- if c.is_active and c.config.get("webhook_channel_id")- ]-- for connection in active_connections:- try:- logger.info(- "Cancelling subscription for connection",- connection_id=connection.connection_id,- )- connector = await connector_service.get_connector(- connection.connection_id- )- if connector:- subscription_id = connection.config.get("webhook_channel_id")- await connector.cleanup_subscription(subscription_id)- logger.info(- "Cancelled subscription", subscription_id=subscription_id- )- except Exception as e:- logger.error(- "Failed to cancel subscription",- connection_id=connection.connection_id,- error=str(e),- )-- logger.info(- "Finished cancelling subscriptions",- subscription_count=len(active_connections),- )-- except Exception as e:- logger.error("Failed to cleanup subscriptions", error=str(e))if __name__ == "__main__":import uvicorn- # TUI check already handled at top of file- # Register cleanup functionatexit.register(cleanup)# Run Alembic upgrade SYNCHRONOUSLY before the app builds. This@@ -2469,24 +79,20 @@ if __name__ == "__main__":# live uvicorn loop.try:from db.migrations_runtime import run_alembic_upgrade+run_alembic_upgrade("head")except Exception as _e:logger.error("Alembic upgrade failed at startup", error=str(_e))raise- # Create app asynchronouslyapp = asyncio.run(create_app())- # Enable or disable HTTP access logging events- access_log = os.getenv("ACCESS_LOG", "true").lower() == "true"-- # Run the server (startup tasks now handled by FastAPI startup event)uvicorn.run(app,workers=1,host="0.0.0.0",port=8000,- reload=False, # Disable reload since we're running from main- access_log=access_log,- log_config=None, # Prevent uvicorn from overriding our structlog setup+ reload=False,+ access_log=ACCESS_LOG_ENABLED,+ log_config=None,)diff --git a/src/services/default_docs_service.py b/src/services/default_docs_service.pynew file mode 100644index 000000000..6a01a412e--- /dev/null+++ b/src/services/default_docs_service.py@@ -0,0 +1,682 @@+"""Default OpenRAG docs ingestion / refresh / upgrade-reingest.++This module owns the bundled-docs onboarding flow: deciding whether to+ingest from a URL or from local files, choosing the Langflow or+direct-OpenRAG pipeline, detecting upstream content changes via HTTP+ETag/Last-Modified, and deleting stale chunks before reingestion.++Public entry points used outside this module:+- ingest_default_documents_when_ready (api/settings.py)+- refresh_default_openrag_docs (api/settings.py)+- _get_remote_docs_signature (test_main_docs_signature.py)+"""++import hashlib+import os++import httpx++from config.paths import get_documents_path+from config.settings import (+ DEFAULT_DOCS_CRAWL_DEPTH,+ DEFAULT_DOCS_INGEST_SOURCE,+ DEFAULT_DOCS_URL,+ DISABLE_INGEST_WITH_LANGFLOW,+ LANGFLOW_URL_INGEST_FLOW_ID,+ config_manager,+ get_index_name,+ get_openrag_config,+)+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClient+from utils.url_content_fetcher import materialize_url_as_text_file+from utils.version_utils import OPENRAG_VERSION++logger = get_logger(__name__)++# Files to exclude from startup ingestion+EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}+URL_INGEST_EXCLUDED_INGESTION_FILES = {"openrag-documentation.pdf"}+++def _get_documents_dir():+ """Get the documents directory path, handling both Docker and local environments."""+ path = get_documents_path()+ logger.debug(f"Using documents path: {path}")+ return path+++def _should_use_url_default_docs_ingest() -> bool:+ """Return whether default docs ingestion should use URL crawling."""+ return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)+++async def ingest_openrag_docs_when_ready(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ jwt_token=None,+):+ """Ingest OpenRAG docs during onboarding."""+ use_url_ingest = _should_use_url_default_docs_ingest()+ task_id = None+ if use_url_ingest:+ try:+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START+ )+ if DISABLE_INGEST_WITH_LANGFLOW:+ task_id = await _ingest_default_documents_url(+ document_service=document_service,+ models_service=models_service,+ docs_url=DEFAULT_DOCS_URL,+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,+ jwt_token=jwt_token,+ )+ else:+ logger.info(+ "Ingesting default documents using Langflow",+ docs_url=DEFAULT_DOCS_URL,+ )+ task_id = await _ingest_default_documents_url_langflow(+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ task_service=task_service,+ docs_url=DEFAULT_DOCS_URL,+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,+ jwt_token=jwt_token,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_COMPLETE+ )+ except Exception as e:+ logger.error("Default URL documents ingestion failed", error=str(e))+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED+ )+ return task_id+++async def ingest_default_documents_when_ready(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ jwt_token=None,+):+ """Ingest default OpenRAG docs during onboarding."""+ try:+ logger.info(+ "Ingesting default documents when ready",+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START+ )+ task_id = await ingest_openrag_docs_when_ready(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ jwt_token=jwt_token,+ )++ base_dir = _get_documents_dir()+ if not os.path.isdir(base_dir):+ raise FileNotFoundError(f"Default documents directory not found: {base_dir}")++ excluded_files = set(EXCLUDED_INGESTION_FILES)+ if _should_use_url_default_docs_ingest():+ excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)++ file_paths = [+ os.path.join(root, fn)+ for root, _, files in os.walk(base_dir)+ for fn in files+ if fn not in excluded_files+ ]++ if not file_paths:+ raise FileNotFoundError(f"No default documents found in {base_dir}")++ if DISABLE_INGEST_WITH_LANGFLOW:+ new_task_id = await _ingest_default_documents_openrag(+ document_service,+ models_service,+ task_service,+ file_paths,+ existing_task_id=task_id,+ connector_type="local",+ jwt_token=jwt_token,+ )+ task_id = new_task_id or task_id+ else:+ new_task_id = await _ingest_default_documents_langflow(+ langflow_file_service,+ session_manager,+ task_service,+ file_paths,+ existing_task_id=task_id,+ connector_type="local",+ jwt_token=jwt_token,+ )+ task_id = new_task_id or task_id++ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE+ )++ return task_id++ except Exception as e:+ logger.error("Default documents ingestion failed", error=str(e))+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED+ )+ raise+++async def _ingest_default_documents_langflow(+ langflow_file_service,+ session_manager,+ task_service,+ file_paths,+ existing_task_id: str = None,+ connector_type: str = "openrag_docs",+ jwt_token=None,+):+ """Ingest default documents using Langflow upload-ingest-delete pipeline."""++ logger.info(+ "Using Langflow ingestion pipeline for default documents",+ file_count=len(file_paths),+ )++ from session_manager import AnonymousUser++ anonymous_user = AnonymousUser()+ effective_jwt = jwt_token++ if not effective_jwt and session_manager:+ session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ default_tweaks = {+ "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {+ "docs_metadata": [+ {"key": "owner", "value": None},+ {"key": "owner_name", "value": anonymous_user.name},+ {"key": "owner_email", "value": anonymous_user.email},+ {"key": "connector_type", "value": "openrag_docs"},+ {"key": "is_sample_data", "value": "true"},+ ]+ }+ }++ task_id = await task_service.create_langflow_upload_task(+ user_id=None,+ file_paths=file_paths,+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ jwt_token=effective_jwt,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ session_id=None,+ tweaks=default_tweaks,+ settings=None,+ replace_duplicates=True,+ connector_type=connector_type,+ existing_task_id=existing_task_id,+ )++ logger.info(+ "Started Langflow ingestion task for default documents",+ task_id=task_id,+ file_count=len(file_paths),+ )+ return task_id+++async def _ingest_default_documents_url_langflow(+ langflow_file_service,+ session_manager,+ task_service,+ docs_url: str,+ crawl_depth: int,+ jwt_token=None,+):+ """Ingest default URL docs using the Langflow URL ingestion pipeline."""+ if not docs_url:+ raise ValueError("DEFAULT_DOCS_URL is not configured")++ logger.info(+ "Using Langflow URL ingestion pipeline for default documents",+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )++ from session_manager import AnonymousUser++ anonymous_user = AnonymousUser()+ effective_jwt = jwt_token++ if not effective_jwt and session_manager:+ session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ default_tweaks = {+ "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {+ "docs_metadata": [+ {"key": "owner", "value": None},+ {"key": "owner_name", "value": anonymous_user.name},+ {"key": "owner_email", "value": anonymous_user.email},+ {"key": "connector_type", "value": "openrag_docs"},+ {"key": "is_sample_data", "value": "true"},+ ]+ }+ }++ task_id = await task_service.create_langflow_url_upload_task(+ owner_user_id=None,+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ jwt_token=effective_jwt,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ connector_type="openrag_docs",+ tweaks=default_tweaks,+ )++ logger.info(+ "Started Langflow URL ingestion task for default documents",+ task_id=task_id,+ docs_url=docs_url,+ )+ return task_id+++async def _ingest_default_documents_url(+ document_service,+ models_service,+ docs_url: str,+ crawl_depth: int,+ jwt_token=None,+):+ """Ingest default docs from URL using OpenRAG ingestion logic (no Langflow)."""+ if not docs_url:+ raise ValueError("DEFAULT_DOCS_URL is not configured")++ logger.info(+ "Running default URL docs ingestion with OpenRAG processor",+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )+ temp_file_path = await materialize_url_as_text_file(+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )+ try:+ from models.processors import DocumentFileProcessor+ from session_manager import AnonymousUser+ from utils.hash_utils import hash_id++ anonymous_user = AnonymousUser()++ processor = DocumentFileProcessor(+ document_service,+ models_service=models_service,+ owner_user_id=None,+ jwt_token=jwt_token,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ is_sample_data=True,+ connector_type="openrag_docs",+ )+ await processor.process_document_standard(+ file_path=temp_file_path,+ file_hash=hash_id(temp_file_path),+ owner_user_id=None,+ original_filename="openrag-url-default.txt",+ jwt_token=jwt_token,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ file_size=os.path.getsize(temp_file_path),+ connector_type="openrag_docs",+ is_sample_data=True,+ )+ finally:+ try:+ os.unlink(temp_file_path)+ except FileNotFoundError:+ pass+ except Exception as e:+ logger.error(+ "Failed to clean temporary default URL docs file",+ path=temp_file_path,+ error=str(e),+ )+++async def _delete_existing_default_docs(session_manager, connector_type: str):+ """Delete previously ingested default OpenRAG docs before reingestion."""+ from session_manager import AnonymousUser++ if session_manager is None:+ logger.warning(+ "Session manager unavailable; skipping default docs cleanup before reingestion"+ )+ return++ anonymous_user = AnonymousUser()+ effective_jwt = None+ if session_manager:+ session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ opensearch_client = session_manager.get_user_opensearch_client(+ anonymous_user.user_id, effective_jwt+ )+ delete_query = {+ "query": {+ "bool": {+ "should": [+ # URL-based default docs are ingested as system_default and+ # owned by the anonymous onboarding user.+ {+ "bool": {+ "must": [+ {"term": {"connector_type": connector_type}},+ {"term": {"owner_email": anonymous_user.email}},+ ]+ }+ },+ ],+ "minimum_should_match": 1,+ }+ }+ }+ result = await opensearch_client.delete_by_query(+ index=get_index_name(),+ body=delete_query,+ conflicts="proceed",+ )+ logger.info(+ "Deleted existing default OpenRAG docs before reingestion",+ deleted_chunks=result.get("deleted", 0),+ )+++async def _reingest_default_docs_on_upgrade_if_needed(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+):+ """Reingest default OpenRAG docs once when app version changes."""+ config = get_openrag_config()++ previous_version = config.onboarding.openrag_docs_ingested_version+ current_version = OPENRAG_VERSION+ should_reingest = bool(previous_version) and previous_version != current_version++ # Legacy installs may not have a stored docs ingestion version.+ # Use the presence of the OpenRAG docs filter as the signal that docs were+ # already onboarded, independent of whether config.edited is set.+ if not previous_version and config.onboarding.openrag_docs_filter_id:+ should_reingest = True++ if not should_reingest:+ return False++ logger.info(+ "Detected OpenRAG upgrade; reingesting default docs",+ previous_version=previous_version,+ current_version=current_version,+ )+ await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")+ await ingest_openrag_docs_when_ready(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ )+ config.onboarding.openrag_docs_ingested_version = current_version+ if _should_use_url_default_docs_ingest():+ # Refresh signature metadata after upgrade reingestion so startup+ # signature checks don't trigger an immediate duplicate ingest.+ config.onboarding.openrag_docs_remote_signature = await _get_remote_docs_signature(+ DEFAULT_DOCS_URL+ )+ else:+ config.onboarding.openrag_docs_remote_signature = None+ if not config_manager.save_config_file(config):+ logger.warning(+ "Default docs were reingested but failed to persist metadata",+ current_version=current_version,+ signature=config.onboarding.openrag_docs_remote_signature,+ )+ return True+++async def _get_remote_docs_signature(docs_url: str):+ """Get a signature for remote docs to detect content updates."""+ try:+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:+ head_response = await client.head(docs_url)+ if head_response.status_code >= 400:+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()++ etag = (head_response.headers.get("etag") or "").strip()+ last_modified = (head_response.headers.get("last-modified") or "").strip()+ if etag:+ # Prefer ETag when available: it is typically the strongest+ # cache validator and stays stable if extra cache headers+ # appear/disappear without content changes.+ return f"etag={etag}"+ if last_modified:+ return f"last_modified={last_modified}"++ # HEAD has no body. If cache headers are missing, fetch the page body.+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature body fallback",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()+ except Exception as e:+ logger.error(+ "Unable to retrieve remote docs signature",+ docs_url=docs_url,+ error=str(e),+ )+ return None+++async def refresh_default_openrag_docs(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ force: bool = False,+ reason: str = "startup",+):+ """Refresh OpenRAG docs if remote content changed or when forced."""+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_START,+ metadata={"reason": reason, "force": force},+ )+ try:+ if not _should_use_url_default_docs_ingest():+ logger.info(+ "Skipping OpenRAG docs refresh: URL ingestion is not active",+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,+ has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),+ has_docs_url=bool(DEFAULT_DOCS_URL),+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "url_ingestion_inactive",+ },+ )+ return False++ config = get_openrag_config()+ if not config.edited:+ logger.info("Skipping OpenRAG docs refresh: onboarding not completed")+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "onboarding_not_completed",+ },+ )+ return False++ signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)+ if not signature and not force:+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "signature_unavailable",+ },+ )+ return False++ previous_signature = config.onboarding.openrag_docs_remote_signature+ should_refresh = force or (signature is not None and signature != previous_signature)+ if not should_refresh:+ logger.info(+ "OpenRAG docs refresh skipped: remote signature unchanged",+ signature=signature,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "signature_unchanged",+ },+ )+ return False++ logger.info(+ "Refreshing default OpenRAG docs",+ reason=reason,+ force=force,+ previous_signature=previous_signature,+ new_signature=signature,+ )+ await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")+ await ingest_openrag_docs_when_ready(+ document_service,+ models_service,+ task_service,+ langflow_file_service,+ session_manager,+ )+ config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION+ # Keep docs version/signature metadata consistent after a refresh.+ # If signature retrieval failed, persist None explicitly instead of+ # leaving a stale previous signature value.+ config.onboarding.openrag_docs_remote_signature = signature+ if not config_manager.save_config_file(config):+ logger.warning(+ "OpenRAG docs refreshed but failed to persist metadata",+ version=config.onboarding.openrag_docs_ingested_version,+ signature=config.onboarding.openrag_docs_remote_signature,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_COMPLETE,+ metadata={"reason": reason, "force": force},+ )+ return True+ except Exception as e:+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_FAILED,+ metadata={+ "reason": reason,+ "force": force,+ "error_type": type(e).__name__,+ },+ )+ raise+++async def _ingest_default_documents_openrag(+ document_service,+ models_service,+ task_service,+ file_paths,+ connector_type: str = "openrag_docs",+ existing_task_id: str = None,+ jwt_token=None,+):+ """Ingest default documents using traditional OpenRAG processor."""+ logger.info(+ "Using traditional OpenRAG ingestion for default documents",+ file_count=len(file_paths),+ )++ from models.processors import DocumentFileProcessor+ from session_manager import AnonymousUser++ anonymous_user = AnonymousUser()++ processor = DocumentFileProcessor(+ document_service,+ models_service=models_service,+ owner_user_id=None,+ jwt_token=jwt_token,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ is_sample_data=True,+ connector_type=connector_type,+ )++ task_id = await task_service.create_custom_task(+ "anonymous", file_paths, processor, existing_task_id=existing_task_id+ )+ logger.info(+ "Started traditional OpenRAG ingestion task",+ task_id=task_id,+ file_count=len(file_paths),+ )+ return task_iddiff --git a/src/services/startup_orchestrator.py b/src/services/startup_orchestrator.pynew file mode 100644index 000000000..834c0b4bd--- /dev/null+++ b/src/services/startup_orchestrator.py@@ -0,0 +1,190 @@+"""Startup orchestration for the OpenRAG backend.++Runs after services are constructed and the FastAPI lifespan begins.+Bootstraps OpenSearch (when not in IBM auth mode), reingests bundled docs+on upgrade, refreshes them if remote content changed, syncs Langflow MCP+server URLs, and reapplies user settings if Langflow flows were reset.+"""++from config.settings import (+ DISABLE_INGEST_WITH_LANGFLOW,+ FETCH_OPENRAG_DOCS_AT_STARTUP,+ clients,+ get_openrag_config,+)+from services.default_docs_service import (+ _reingest_default_docs_on_upgrade_if_needed,+ refresh_default_openrag_docs,+)+from utils.logging_config import get_logger+from utils.opensearch_init import (+ _ensure_opensearch_index,+ configure_alerting_security,+ init_index,+ wait_for_opensearch,+)+from utils.telemetry import Category, MessageId, TelemetryClient++logger = get_logger(__name__)+++async def _update_mcp_server_urls(langflow_mcp_service):+ """Update MCP server URLs (patch localhost and convert to streamable HTTP)."""+ try:+ result = await langflow_mcp_service.update_all_mcp_server_urls()+ logger.info("Updated MCP server URLs after settings change", **result)+ except Exception as mcp_error:+ logger.warning(f"Failed to update MCP server URLs after settings change: {str(mcp_error)}")+++async def startup_tasks(services):+ """Startup tasks"""+ from config.settings import IBM_AUTH_ENABLED++ logger.info("Starting startup tasks")+ await TelemetryClient.send_event(Category.APPLICATION_STARTUP, MessageId.ORB_APP_START_INIT)++ # Update model registry to allow further search calls to be instant+ try:+ models_service = services["models_service"]+ await models_service.update_model_registry()+ except Exception as e:+ logger.error(+ "Failed to update model registry at startup — "+ "models may be missing until the next restart",+ error=str(e),+ )++ if IBM_AUTH_ENABLED:+ logger.info(+ "IBM auth mode: skipping startup OpenSearch checks. "+ "OpenSearch will be initialized during onboarding with user credentials."+ )+ else:+ # Only initialize basic OpenSearch connection, not the index+ # Index will be created after onboarding when we know the embedding model+ await wait_for_opensearch()++ # Setup OpenSearch security (roles and mappings) after connection is established+ try:+ from utils.opensearch_utils import setup_opensearch_security++ await setup_opensearch_security(clients.opensearch)+ logger.info("OpenSearch security configuration completed successfully")+ except Exception as e:+ logger.warning(+ "Failed to setup OpenSearch security configuration - continuing anyway",+ error=str(e),+ )++ if DISABLE_INGEST_WITH_LANGFLOW:+ await _ensure_opensearch_index()++ # Ensure that the OpenSearch index exists if onboarding was already completed+ # - Handles the case where OpenSearch is reset (e.g., volume deleted) after onboarding+ embedding_model = None+ try:+ config = get_openrag_config()+ embedding_model = config.knowledge.embedding_model++ if config.edited and embedding_model:+ logger.info(+ "Ensuring that the OpenSearch index exists (after onboarding)...",+ embedding_model=embedding_model,+ )++ await init_index()++ logger.info(+ "Successfully ensured that the OpenSearch index exists (after onboarding).",+ embedding_model=embedding_model,+ )+ except Exception as e:+ logger.error(+ "Failed to ensure that the OpenSearch index exists (after onboarding).",+ embedding_model=embedding_model,+ error=str(e),+ )+ raise++ await configure_alerting_security()++ # Reingest bundled OpenRAG docs once after application upgrade.+ upgrade_reingested = False+ try:+ upgrade_reingested = await _reingest_default_docs_on_upgrade_if_needed(+ services["document_service"],+ services["models_service"],+ services["task_service"],+ services["langflow_file_service"],+ services["session_manager"],+ )+ except Exception as e:+ logger.error("Default docs reingestion on upgrade failed", error=str(e))++ if FETCH_OPENRAG_DOCS_AT_STARTUP and not upgrade_reingested:+ try:+ await refresh_default_openrag_docs(+ services["document_service"],+ services["models_service"],+ services["task_service"],+ services["langflow_file_service"],+ services["session_manager"],+ force=False,+ reason="startup",+ )+ except Exception as e:+ logger.error("OpenRAG docs startup refresh failed", error=str(e))++ # Update MCP server URLs (patch localhost and convert to streamable HTTP)+ await _update_mcp_server_urls(services["langflow_mcp_service"])++ # Ensure all configured flows exist in Langflow (create-only, never overwrites).+ # This replaces LANGFLOW_LOAD_FLOWS_PATH, which performed a blind upsert on+ # every container start and discarded any user edits made in the Langflow UI.+ newly_created: set[str] = set()+ try:+ flows_service = services["flows_service"]+ newly_created = await flows_service.ensure_flows_exist()+ except Exception as e:+ logger.error(+ "Failed to ensure Langflow flows exist at startup — "+ "flows may be missing until the next restart",+ error=str(e),+ )++ # Check if flows were reset and reapply settings if config is edited+ try:+ config = get_openrag_config()+ if config.edited:+ logger.info("Checking if Langflow flows were reset")+ flows_service = services["flows_service"]+ reset_flows = await flows_service.check_flows_reset()+ # Exclude flows that were just seeded — they match the JSON by design,+ # not because they were externally reset.+ reset_flows = [f for f in reset_flows if f not in newly_created]++ if reset_flows:+ logger.info(+ f"Detected reset flows: {', '.join(reset_flows)}. Reapplying all settings."+ )+ await TelemetryClient.send_event(+ Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_RESET_DETECTED+ )+ from api.settings import reapply_all_settings++ await reapply_all_settings(session_manager=services["session_manager"])+ logger.info("Successfully reapplied settings after detecting flow resets")+ await TelemetryClient.send_event(+ Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_SETTINGS_REAPPLIED+ )+ else:+ logger.info("No flows detected as reset, skipping settings reapplication")+ else:+ logger.debug("Configuration not yet edited, skipping flow reset check")+ except Exception as e:+ logger.error(f"Failed to check flows reset or reapply settings: {str(e)}")+ await TelemetryClient.send_event(+ Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_RESET_CHECK_FAIL+ )+ # Don't fail startup if this check failsdiff --git a/src/utils/jwt_keygen.py b/src/utils/jwt_keygen.pynew file mode 100644index 000000000..392d07230--- /dev/null+++ b/src/utils/jwt_keygen.py@@ -0,0 +1,55 @@+"""RSA keypair generation for JWT signing."""++import os+import subprocess++from config.paths import get_keys_path+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClient++logger = get_logger(__name__)+++def generate_jwt_keys():+ """Generate RSA keys for JWT signing if they don't exist"""+ keys_dir = get_keys_path()+ private_key_path = os.path.join(keys_dir, "private_key.pem")+ public_key_path = os.path.join(keys_dir, "public_key.pem")++ os.makedirs(keys_dir, exist_ok=True)++ if not os.path.exists(private_key_path):+ try:+ subprocess.run(+ ["openssl", "genrsa", "-out", private_key_path, "2048"],+ check=True,+ capture_output=True,+ )++ os.chmod(private_key_path, 0o600)++ subprocess.run(+ [+ "openssl",+ "rsa",+ "-in",+ private_key_path,+ "-pubout",+ "-out",+ public_key_path,+ ],+ check=True,+ capture_output=True,+ )++ os.chmod(public_key_path, 0o644)++ logger.info("Generated RSA keys for JWT signing")+ except subprocess.CalledProcessError as e:+ logger.error("Failed to generate RSA keys", error=str(e))+ TelemetryClient.send_event_sync(+ Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_JWT_KEY_FAIL+ )+ raise+ else:+ logger.info("RSA keys already exist")diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.pynew file mode 100644index 000000000..95bcdd503--- /dev/null+++ b/src/utils/opensearch_init.py@@ -0,0 +1,209 @@+"""OpenSearch index/security initialization at startup.++Pure functions that create or update OpenSearch indices and apply security+configuration. Imported by app startup orchestration and by api/* modules+that need to (re)create the documents index after onboarding completes.+"""++from config.settings import (+ API_KEYS_INDEX_BODY,+ API_KEYS_INDEX_NAME,+ INDEX_BODY,+ clients,+ get_index_name,+ get_openrag_config,+)+from utils.embeddings import create_index_body+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClient++logger = get_logger(__name__)+++async def wait_for_opensearch(opensearch_client=None):+ """Wait for OpenSearch to be ready, delegating to the shared utility."""+ from utils.opensearch_utils import (+ OpenSearchNotReadyError,+ )+ from utils.opensearch_utils import (+ wait_for_opensearch as _wait_for_opensearch,+ )++ try:+ await _wait_for_opensearch(opensearch_client or clients.opensearch)+ await TelemetryClient.send_event(+ Category.OPENSEARCH_SETUP, MessageId.ORB_OS_CONN_ESTABLISHED+ )+ except OpenSearchNotReadyError:+ await TelemetryClient.send_event(Category.OPENSEARCH_SETUP, MessageId.ORB_OS_TIMEOUT)+ raise+++async def configure_alerting_security():+ """Configure OpenSearch alerting plugin security settings"""+ try:+ alerting_settings = {+ "persistent": {+ "plugins.alerting.filter_by_backend_roles": "false",+ "opendistro.alerting.filter_by_backend_roles": "false",+ "opensearch.notifications.general.filter_by_backend_roles": "false",+ }+ }++ response = await clients.opensearch.cluster.put_settings(body=alerting_settings)+ logger.info("Alerting security settings configured successfully", response=response)+ except Exception as e:+ logger.error("Failed to configure alerting security settings", error=str(e))+++async def _ensure_opensearch_index():+ """Ensure OpenSearch index exists when using traditional connector service."""+ try:+ index_name = get_index_name()+ if await clients.opensearch.indices.exists(index=index_name):+ logger.info("[OPENSEARCH] Index already exists", index_name=index_name)+ return++ await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)+ logger.info(+ "Created OpenSearch index for traditional connector service",+ index_name=index_name,+ vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"]["dimension"],+ )+ await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED)++ except Exception as e:+ logger.error(+ "Failed to initialize OpenSearch index for traditional connector service",+ error=str(e),+ index_name=get_index_name(),+ )+ await TelemetryClient.send_event(+ Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL+ )+++async def init_index(opensearch_client=None, admin_username: str = None):+ """Initialize OpenSearch index and security roles"""+ os_client = opensearch_client or clients.opensearch+ try:+ await wait_for_opensearch(opensearch_client)++ from utils.opensearch_utils import setup_opensearch_security++ await setup_opensearch_security(os_client, admin_username=admin_username)++ config = get_openrag_config()+ embedding_model = config.knowledge.embedding_model++ index_body = await create_index_body()++ index_name = get_index_name()+ if not await os_client.indices.exists(index=index_name):+ await os_client.indices.create(index=index_name, body=index_body)+ logger.info(+ "Created OpenSearch index",+ index_name=index_name,+ embedding_model=embedding_model,+ )+ await TelemetryClient.send_event(+ Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED+ )+ else:+ logger.info(+ "Index already exists, skipping creation and changing number of replicas",+ index_name=index_name,+ embedding_model=embedding_model,+ )+ current = await os_client.indices.get_settings(index=index_name)+ current_replicas = int(+ current[index_name]["settings"]["index"].get("number_of_replicas", 1)+ )+ if current_replicas != 0:+ await os_client.indices.put_settings(+ index=index_name,+ body={"index": {"number_of_replicas": 0}},+ )+ logger.info(+ "Updated documents index settings",+ )+ await TelemetryClient.send_event(+ Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS+ )++ knowledge_filter_index_name = "knowledge_filters"+ knowledge_filter_index_body = {+ "settings": {+ "index": {"number_of_replicas": 0, "number_of_shards": 1},+ },+ "mappings": {+ "properties": {+ "id": {"type": "keyword"},+ "name": {"type": "text", "analyzer": "standard"},+ "description": {"type": "text", "analyzer": "standard"},+ "query_data": {"type": "text"},+ "owner": {"type": "keyword"},+ "allowed_users": {"type": "keyword"},+ "allowed_groups": {"type": "keyword"},+ "subscriptions": {"type": "object"},+ "created_at": {"type": "date"},+ "updated_at": {"type": "date"},+ }+ },+ }++ if not await os_client.indices.exists(index=knowledge_filter_index_name):+ await os_client.indices.create(+ index=knowledge_filter_index_name, body=knowledge_filter_index_body+ )+ logger.info(+ "Created knowledge filters index",+ index_name=knowledge_filter_index_name,+ )+ await TelemetryClient.send_event(+ Category.OPENSEARCH_INDEX, MessageId.ORB_OS_KF_INDEX_CREATED+ )+ else:+ logger.info(+ "Knowledge filters index already exists, skipping creation",+ index_name=knowledge_filter_index_name,+ )++ current = await os_client.indices.get_settings(index=knowledge_filter_index_name)+ current_replicas = int(+ current[knowledge_filter_index_name]["settings"]["index"].get(+ "number_of_replicas", 1+ )+ )+ if current_replicas != 0:+ await os_client.indices.put_settings(+ index=knowledge_filter_index_name,+ body={"index": {"number_of_replicas": 0}},+ )+ logger.info(+ "Updated knowledge filters index settings",+ )++ if not await os_client.indices.exists(index=API_KEYS_INDEX_NAME):+ await os_client.indices.create(index=API_KEYS_INDEX_NAME, body=API_KEYS_INDEX_BODY)+ logger.info("Created API keys index", index_name=API_KEYS_INDEX_NAME)+ else:+ logger.info(+ "API keys index already exists, skipping creation",+ index_name=API_KEYS_INDEX_NAME,+ )++ await configure_alerting_security()++ except Exception as e:+ from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error++ if is_disk_space_error(e):+ logger.error(+ "OpenSearch disk space exceeded watermark. Index creation failed."+ )+ raise OpenSearchDiskSpaceError(+ "OpenSearch disk space is full (watermark exceeded). "+ "Please free up disk space on your Docker volume or host machine to continue."+ ) from e+ raisediff --git a/src/utils/url_content_fetcher.py b/src/utils/url_content_fetcher.pynew file mode 100644index 000000000..e9bfb8071--- /dev/null+++ b/src/utils/url_content_fetcher.py@@ -0,0 +1,71 @@+"""Fetch a URL, strip HTML, and write the visible text to a temporary file.++Used by the default-docs ingestion flow when DEFAULT_DOCS_INGEST_SOURCE=url+and the OpenRAG (non-Langflow) ingestion path is active.+"""++import html+import re+import tempfile+from html.parser import HTMLParser++import httpx+++class _VisibleTextHTMLParser(HTMLParser):+ """Extract visible text while skipping script/style content."""++ def __init__(self):+ super().__init__(convert_charrefs=True)+ self._ignored_depth = 0+ self._chunks: list[str] = []++ def handle_starttag(self, tag, attrs):+ if tag.lower() in {"script", "style"}:+ self._ignored_depth += 1++ def handle_endtag(self, tag):+ if tag.lower() in {"script", "style"} and self._ignored_depth > 0:+ self._ignored_depth -= 1++ def handle_data(self, data):+ if self._ignored_depth == 0 and data and not data.isspace():+ self._chunks.append(data)++ def get_text(self) -> str:+ return " ".join(self._chunks)+++async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:+ """Fetch URL content and write a temporary text file for OpenRAG ingestion."""+ async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:+ response = await client.get(docs_url)+ response.raise_for_status()+ raw_html = response.text++ title_match = re.search(+ r"<title[^>]*>(.*?)</title\s*>",+ raw_html,+ flags=re.IGNORECASE | re.DOTALL,+ )+ title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"++ text_parser = _VisibleTextHTMLParser()+ text_parser.feed(raw_html)+ text_parser.close()+ normalized_text = re.sub(r"\s+", " ", text_parser.get_text()).strip()++ content = (+ f"{title}\n\nSource URL: {docs_url}\nCrawl depth: {crawl_depth}\n\n{normalized_text}\n"+ )++ temp_file = tempfile.NamedTemporaryFile(+ mode="w",+ suffix=".txt",+ prefix="openrag-url-default-",+ delete=False,+ encoding="utf-8",+ )+ with temp_file:+ temp_file.write(content)+ return temp_file.namediff --git a/tests/integration/core/test_api_endpoints.py b/tests/integration/core/test_api_endpoints.pyindex 9c0edd212..37b0258a8 100644--- a/tests/integration/core/test_api_endpoints.py+++ b/tests/integration/core/test_api_endpoints.py@@ -11,9 +11,17 @@ def dump_docker_logs(container_name_pattern: str = "langflow", tail: int = 100):"""Dump Docker container logs for debugging."""try:# Find container ID by name pattern- find_cmd = ["docker", "ps", "-a", "--filter", f"name={container_name_pattern}", "--format", "{{.ID}}"]+ find_cmd = [+ "docker",+ "ps",+ "-a",+ "--filter",+ f"name={container_name_pattern}",+ "--format",+ "{{.ID}}",+ ]result = subprocess.run(find_cmd, capture_output=True, text=True, timeout=5)- container_ids = result.stdout.strip().split('\n')+ container_ids = result.stdout.strip().split("\n")if not container_ids or not container_ids[0]:print(f"[DEBUG] No Docker containers found matching pattern: {container_name_pattern}")@@ -22,16 +30,16 @@ def dump_docker_logs(container_name_pattern: str = "langflow", tail: int = 100):for container_id in container_ids:if not container_id:continue- print(f"\n{'='*80}")+ print(f"\n{'=' * 80}")print(f"[DEBUG] Docker logs for container {container_id} (last {tail} lines):")- print(f"{'='*80}")+ print(f"{'=' * 80}")logs_cmd = ["docker", "logs", "--tail", str(tail), container_id]logs_result = subprocess.run(logs_cmd, capture_output=True, text=True, timeout=10)print(logs_result.stdout)if logs_result.stderr:print("[STDERR]:", logs_result.stderr)- print(f"{'='*80}\n")+ print(f"{'=' * 80}\n")except subprocess.TimeoutExpired:print(f"[DEBUG] Timeout while fetching docker logs for {container_name_pattern}")except Exception as e:@@ -46,21 +54,26 @@ async def wait_for_service_ready(client: httpx.AsyncClient, timeout_s: float = 3- POST /search with query "*" avoids embeddings and checks OpenSearch/index readiness."""# First test OpenSearch JWT directly- from session_manager import SessionManager, AnonymousUser- import osimport hashlib+ import os+import jwt as jwt_lib++ from session_manager import AnonymousUser, SessionManager+sm = SessionManager("test")test_token = sm.create_jwt_token(AnonymousUser())raw_token = test_token.removeprefix("Bearer ")token_hash = hashlib.sha256(test_token.encode()).hexdigest()[:16]print(f"[DEBUG] Generated test JWT token hash: {token_hash}")print(f"[DEBUG] Using key paths: private={sm.private_key_path}, public={sm.public_key_path}")- with open(sm.public_key_path, 'rb') as f:+ with open(sm.public_key_path, "rb") as f:pub_key_hash = hashlib.sha256(f.read()).hexdigest()[:16]print(f"[DEBUG] Public key hash: {pub_key_hash}")decoded = jwt_lib.decode(raw_token, options={"verify_signature": False})- print(f"[DEBUG] JWT claims: iss={decoded.get('iss')}, sub={decoded.get('sub')}, aud={decoded.get('aud')}, roles={decoded.get('roles')}")+ print(+ f"[DEBUG] JWT claims: iss={decoded.get('iss')}, sub={decoded.get('sub')}, aud={decoded.get('aud')}, roles={decoded.get('roles')}"+ )# Test OpenSearch JWT auth directlyopensearch_url = f"https://{os.getenv('OPENSEARCH_HOST', 'localhost')}:{os.getenv('OPENSEARCH_PORT', '9200')}"@@ -69,13 +82,15 @@ async def wait_for_service_ready(client: httpx.AsyncClient, timeout_s: float = 3r_os = await os_client.post(f"{opensearch_url}/documents/_search",headers={"Authorization": f"Bearer {raw_token}"},- json={"query": {"match_all": {}}, "size": 0}+ json={"query": {"match_all": {}}, "size": 0},+ )+ print(+ f"[DEBUG] Direct OpenSearch JWT test: status={r_os.status_code}, body={r_os.text[:500]}")- print(f"[DEBUG] Direct OpenSearch JWT test: status={r_os.status_code}, body={r_os.text[:500]}")if r_os.status_code == 401:- print(f"[DEBUG] ❌ OpenSearch rejected JWT! OIDC config not working.")+ print("[DEBUG] ❌ OpenSearch rejected JWT! OIDC config not working.")else:- print(f"[DEBUG] ✓ OpenSearch accepted JWT!")+ print("[DEBUG] ✓ OpenSearch accepted JWT!")deadline = asyncio.get_event_loop().time() + timeout_slast_err = None@@ -122,7 +137,21 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges# Import after env vars to ensure settings pick them up. Clear cached modulesimport sys- # Clear cached modules so settings pick up env and router sees new flag++ # Pop these from sys.modules so the next `from main import ...`+ # forces a fresh import chain after we mutate os.environ above.+ # The list covers two cases:+ # 1. Modules that read env-driven constants from config.settings+ # at module top (e.g. DISABLE_INGEST_WITH_LANGFLOW). Without+ # popping, the bound value is stale.+ # 2. Modules that cache function/class refs from (1) at module+ # top (e.g. app.routes.internal -> api.router, or+ # app.lifespan -> services.startup_orchestrator). Without+ # popping, callers see the old reference even though the+ # underlying module was reloaded fresh.+ # Future maintainers: when adding an app.* or services.* module+ # that does either of the above, add it to ALL four pop lists in+ # this file.for mod in ["api.router","api.connector_router",@@ -132,11 +161,16 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges"api","services","services.search_service",+ "services.default_docs_service",+ "services.startup_orchestrator",+ "app.routes.internal",+ "app.routes",+ "app.factory",+ "app.lifespan",]:sys.modules.pop(mod, None)+ from config.settings import clients, get_index_namefrom main import create_app, startup_tasks- import api.router as upload_router- from config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW# Ensure a clean index before startupawait clients.initialize()@@ -153,14 +187,17 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges# Ensure index exists for tests (startup_tasks only creates it if DISABLE_INGEST_WITH_LANGFLOW=True)from main import _ensure_opensearch_index+await _ensure_opensearch_index()# Verify index is truly empty after startuptry:count_response = await clients.opensearch.count(index=get_index_name())- doc_count = count_response.get('count', 0)- assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents"- except Exception as e:+ doc_count = count_response.get("count", 0)+ assert doc_count == 0, (+ f"Index should be empty after startup but contains {doc_count} documents"+ )+ except Exception:# If count fails, the index might not exist yet, which is finepass@@ -236,6 +273,7 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_ingesfinally:# Explicitly close global clients to avoid aiohttp warningsfrom config.settings import clients+try:await clients.close()except Exception:@@ -264,9 +302,11 @@ async def _wait_for_langflow_chat(await asyncio.sleep(1.0)# Dump Langflow logs before raising error- print(f"\n[DEBUG] /langflow timed out. Dumping Langflow container logs...")+ print("\n[DEBUG] /langflow timed out. Dumping Langflow container logs...")dump_docker_logs(container_name_pattern="langflow", tail=200)- raise AssertionError(f"/langflow never returned a usable response. Last payload: {last_payload}")+ raise AssertionError(+ f"/langflow never returned a usable response. Last payload: {last_payload}"+ )async def _wait_for_nudges(@@ -294,7 +334,9 @@ async def _wait_for_nudges(# Dump Langflow logs before raising errorprint(f"\n[DEBUG] {endpoint} timed out. Dumping Langflow container logs...")dump_docker_logs(container_name_pattern="langflow", tail=200)- raise AssertionError(f"{endpoint} never returned a usable response. Last payload: {last_payload}")+ raise AssertionError(+ f"{endpoint} never returned a usable response. Last payload: {last_payload}"+ )async def _wait_for_task_completion(@@ -321,9 +363,7 @@ async def _wait_for_task_completion(else:last_payload = resp.textawait asyncio.sleep(1.0)- raise AssertionError(- f"Task {task_id} did not complete in time. Last payload: {last_payload}"- )+ raise AssertionError(f"Task {task_id} did not complete in time. Last payload: {last_payload}")@pytest.mark.asyncio@@ -352,12 +392,17 @@ async def test_langflow_chat_and_nudges_endpoints():"api","services","services.search_service",+ "services.default_docs_service",+ "services.startup_orchestrator",+ "app.routes.internal",+ "app.routes",+ "app.factory","services.chat_service",]:sys.modules.pop(mod, None)+ from config.settings import LANGFLOW_CHAT_FLOW_ID, NUDGES_FLOW_ID, clientsfrom main import create_app, startup_tasks- from config.settings import clients, LANGFLOW_CHAT_FLOW_ID, NUDGES_FLOW_IDassert LANGFLOW_CHAT_FLOW_ID, "LANGFLOW_CHAT_FLOW_ID must be configured for integration test"assert NUDGES_FLOW_ID, "NUDGES_FLOW_ID must be configured for integration test"@@ -373,7 +418,9 @@ async def test_langflow_chat_and_nudges_endpoints():if langflow_client is not None:breakawait asyncio.sleep(1.0)- assert langflow_client is not None, "Langflow client not initialized. Provide LANGFLOW_KEY or enable superuser auto-login."+ assert langflow_client is not None, (+ "Langflow client not initialized. Provide LANGFLOW_KEY or enable superuser auto-login."+ )transport = httpx.ASGITransport(app=app)try:@@ -436,9 +483,7 @@ async def test_langflow_chat_and_nudges_endpoints():@pytest.mark.skip@pytest.mark.asyncio-async def test_search_multi_embedding_models(- tmp_path: Path-):+async def test_search_multi_embedding_models(tmp_path: Path):"""Ensure /search fans out across multiple embedding models when present."""os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = "true"os.environ["DISABLE_STARTUP_INGEST"] = "true"@@ -454,11 +499,17 @@ async def test_search_multi_embedding_models("auth_middleware","main","services.search_service",+ "services.default_docs_service",+ "services.startup_orchestrator",+ "app.routes.internal",+ "app.routes",+ "app.factory",+ "app.lifespan",]:sys.modules.pop(mod, None)- from main import create_app, startup_tasksfrom config.settings import clients, get_index_name+ from main import create_app, startup_tasksawait clients.initialize()try:@@ -520,9 +571,7 @@ async def test_search_multi_embedding_models(return payloadlast_payload = payloadawait asyncio.sleep(0.5)- raise AssertionError(- f"Embedding models not detected. Last payload: {last_payload}"- )+ raise AssertionError(f"Embedding models not detected. Last payload: {last_payload}")# Start with explicit small embedding modelresp = await client.post(@@ -584,6 +633,7 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflowos.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""import sys+for mod in ["api.router","api.connector_router",@@ -593,11 +643,16 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow"api","services","services.search_service",+ "services.default_docs_service",+ "services.startup_orchestrator",+ "app.routes.internal",+ "app.routes",+ "app.factory",+ "app.lifespan",]:sys.modules.pop(mod, None)+ from config.settings import clients, get_index_namefrom main import create_app, startup_tasks- import api.router as upload_router- from config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW# Ensure a clean index before startupawait clients.initialize()@@ -613,14 +668,17 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow# Ensure index exists for tests (startup_tasks only creates it if DISABLE_INGEST_WITH_LANGFLOW=True)from main import _ensure_opensearch_index+await _ensure_opensearch_index()# Verify index is truly empty after startuptry:count_response = await clients.opensearch.count(index=get_index_name())- doc_count = count_response.get('count', 0)- assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents"- except Exception as e:+ doc_count = count_response.get("count", 0)+ assert doc_count == 0, (+ f"Index should be empty after startup but contains {doc_count} documents"+ )+ except Exception:# If count fails, the index might not exist yet, which is finepasstransport = httpx.ASGITransport(app=app)@@ -653,6 +711,7 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflowassert data.get("file_count") == 1finally:from config.settings import clients+try:await clients.close()except Exception:diff --git a/uv.lock b/uv.lockindex 986481000..f90b777aa 100644--- a/uv.lock+++ b/uv.lock@@ -2,16 +2,21 @@ version = 1revision = 3requires-python = ">=3.13"resolution-markers = [- "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'",+ "python_full_version >= '3.15' and platform_machine == 'x86_64' and sys_platform == 'linux'",+ "python_full_version == '3.14.*' and platform_machine == 'x86_64' and sys_platform == 'linux'","python_full_version < '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'",- "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'",+ "python_full_version >= '3.15' and platform_machine == 'aarch64' and sys_platform == 'linux'",+ "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'","python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'","python_version < '0'",- "python_full_version >= '3.14' and sys_platform == 'win32'",+ "python_full_version >= '3.15' and sys_platform == 'win32'",+ "python_full_version == '3.14.*' and sys_platform == 'win32'","python_full_version < '3.14' and sys_platform == 'win32'",- "(python_full_version >= '3.14' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')",+ "(python_full_version >= '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.15' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')",+ "(python_full_version == '3.14.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.14.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')","(python_full_version < '3.14' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')",- "python_full_version >= '3.14' and sys_platform == 'darwin'",+ "python_full_version >= '3.15' and sys_platform == 'darwin'",+ "python_full_version == '3.14.*' and sys_platform == 'darwin'","python_full_version < '3.14' and sys_platform == 'darwin'",]@@ -199,6 +204,44 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },]+[[package]]+name = "ast-serialize"+version = "0.3.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/a9/9d/912fefab0e30aee6a3af8a62bbea4a81b29afa4ba2c973d31170620a26de/ast_serialize-0.3.0.tar.gz", hash = "sha256:1bc3ca09a63a021376527c4e938deedd11d11d675ce850e6f9c7487f5889992b", size = 60689, upload-time = "2026-04-30T23:24:48.104Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/6f/57/a54d4de491d6cdd7a4e4b0952cc3ca9f60dcefa7b5fb48d6d492debe1649/ast_serialize-0.3.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a867927df59f76a18dc1d874a0b2c079b42c58972dca637905576deb0912e14", size = 1182966, upload-time = "2026-04-30T23:23:57.376Z" },+ { url = "https://files.pythonhosted.org/packages/ee/9e/a5db014bb0f91b209236b57c429389e31290c0093532b8436d577699b2fa/ast_serialize-0.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a6fb063bf040abf8321e7b8113a0554eda445ffc508aa51287f8808886a5ae22", size = 1171316, upload-time = "2026-04-30T23:23:59.63Z" },+ { url = "https://files.pythonhosted.org/packages/15/59/fd55133e478c4326f60a11df02573bf7ccb2ac685810b50f1803d0f68053/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5075cd8482573d743586779e5f9b652a015e37d4e95132d7e5a9bc5c8f483d8f", size = 1232234, upload-time = "2026-04-30T23:24:01.168Z" },+ { url = "https://files.pythonhosted.org/packages/cc/79/0ca1d26357ecb4a697d74d00b73ef3137f24c140424125393a0de820eb09/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:41560b27794f4553b0f77811e9fb325b77db4a2b39018d437e09932275306e66", size = 1233437, upload-time = "2026-04-30T23:24:03.151Z" },+ { url = "https://files.pythonhosted.org/packages/53/3e/7078ec94dd6e124b8e028ac77016a4f13c83fa1c145790f2e68f3816998b/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b967c01ca74909c5d90e0fe4393401e2cc5da5ebd9a6262a19e45ffd3757dec8", size = 1440188, upload-time = "2026-04-30T23:24:04.717Z" },+ { url = "https://files.pythonhosted.org/packages/21/16/cca7195ef55a012f8013c3442afa91d287a0a36dcf88b480b262475135b3/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:424ebb8f46cd993f7cec4009d119312d8433dd90e6b0df0499cd2c91bdcc5af9", size = 1254211, upload-time = "2026-04-30T23:24:06.18Z" },+ { url = "https://files.pythonhosted.org/packages/a0/0f/f3d4dfae67dee6580534361a6343367d34217e7d25cff858bd1d8f03b8ed/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d14b1d566b56e2ee70b11fec1de7e0b94ec7cd83717ec7d189967841a361190e", size = 1255973, upload-time = "2026-04-30T23:24:07.772Z" },+ { url = "https://files.pythonhosted.org/packages/14/41/55fbfe02c42f40fbe3e74eda167d977d555ff720ce1abfa08515236efd88/ast_serialize-0.3.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7ba30b18735f047ec11103d1ab92f4789cf1fea1e0dc89b04a2f5a0632fd79de", size = 1298629, upload-time = "2026-04-30T23:24:09.4Z" },+ { url = "https://files.pythonhosted.org/packages/28/36/7d2501cacc7989fb8504aa9da2a2022a174200a59d4e6639de4367a57fdd/ast_serialize-0.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e6ea0754cb7b0f682ebb005ffb0d18f8d17993490d9c289863cd69cacc4ab8df", size = 1408435, upload-time = "2026-04-30T23:24:11.013Z" },+ { url = "https://files.pythonhosted.org/packages/03/e7/54e3b469c3fa0bf9cd532fa643d1d33b73303f8d70beac3e366b68dd64b7/ast_serialize-0.3.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:a0c5aa1073a5ba7b2abaa4b54abe8b8d75c4d1e2d54a2ff70b0ca6222fea5728", size = 1508174, upload-time = "2026-04-30T23:24:12.635Z" },+ { url = "https://files.pythonhosted.org/packages/b5/2a/9b9621865b02c60539e26d9b114a312b4fa46aa703e33e79317174bfea21/ast_serialize-0.3.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4e52650d834c1ea7791969a361de2c54c13b2fb4c519ec79445fa8b9021a147d", size = 1502354, upload-time = "2026-04-30T23:24:14.186Z" },+ { url = "https://files.pythonhosted.org/packages/34/dd/f138bc5c43b0c414fdd12eefe15677839323078b6e75301ad7f96cd26d45/ast_serialize-0.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15bd6af3f136c61dae27805eb6b8f3269e85a545c4c27ffe9e530ead78d2b36d", size = 1450504, upload-time = "2026-04-30T23:24:16.076Z" },+ { url = "https://files.pythonhosted.org/packages/68/cf/97ef9e1c315601db74365955c8edd3292e3055500d6317602815dbdf08ae/ast_serialize-0.3.0-cp314-cp314t-win32.whl", hash = "sha256:d188bfe37b674b49708497683051d4b571366a668799c9b8e8a94513694969d9", size = 1058662, upload-time = "2026-04-30T23:24:17.535Z" },+ { url = "https://files.pythonhosted.org/packages/f8/d6/e2c3483c31580fdb623f92ad38d2f856cde4b9205a3e6bd84760f3de7d82/ast_serialize-0.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5832c2fdf8f8a6cf682b4cfcf677f5eaf39b4ddbc490f5480cfccdd1e7ce8fa1", size = 1100349, upload-time = "2026-04-30T23:24:18.992Z" },+ { url = "https://files.pythonhosted.org/packages/ab/89/29abcb1fe18a429cda60c6e0bbd1d6e90499339842a2f548d7567542357e/ast_serialize-0.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:670f177188d128fb7f9f15b5ad0e1b553d22c34e3f584dcb83eb8077600437f0", size = 1072895, upload-time = "2026-04-30T23:24:20.706Z" },+ { url = "https://files.pythonhosted.org/packages/bc/93/72abad83966ed6235647c9f956417dc1e17e997696388521910e3d1fa3f4/ast_serialize-0.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ec2fafa5e4313cc8feed96e436ebe19ac7bc6fa41fbc2827e826c48b9e4c3a9", size = 1190024, upload-time = "2026-04-30T23:24:22.486Z" },+ { url = "https://files.pythonhosted.org/packages/85/4f/eb88584b2f0234e581762011208ca203252bf6c98e59b4769daa571f3576/ast_serialize-0.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ef6d3c08b7b4cd29b48410338e134764a00e76d25841eb02c1084e868c888ecc", size = 1178633, upload-time = "2026-04-30T23:24:24.35Z" },+ { url = "https://files.pythonhosted.org/packages/56/51/cf1ec1ff3e616373d0dcbd5fad502e0029dc541f13ab642259762a7d127f/ast_serialize-0.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d841424f41b886e98044abc80769c14a956e6e5ccd5fb5b0d9f5ead72be18a4", size = 1241351, upload-time = "2026-04-30T23:24:25.987Z" },+ { url = "https://files.pythonhosted.org/packages/0d/44/68fcf50478cf1093f2d423f034ae06453122c8b415d8e21a44668eca485d/ast_serialize-0.3.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d21453734ad39367ede5d37efe4f59f830ce1c09f432fc72a90e368f77a4a3e7", size = 1239582, upload-time = "2026-04-30T23:24:27.808Z" },+ { url = "https://files.pythonhosted.org/packages/9d/c1/a6c9fa284eceb5fc6f21347e968445a051d7ca2c4d34e6a04314646dbcee/ast_serialize-0.3.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5e110cdce2a347e1dd987529c88ef54d26f67848dce3eba1b3b2cc2cf085c94", size = 1448853, upload-time = "2026-04-30T23:24:29.534Z" },+ { url = "https://files.pythonhosted.org/packages/23/5f/8ad3829a09e4e8c5328a53ce7d4711d660944e3e164c5f6abcc2c8f27167/ast_serialize-0.3.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b6e23a98e57560a055f5c4b68700a0fd5ce483d2814c23140b3638c7f5d1e61", size = 1262204, upload-time = "2026-04-30T23:24:31.482Z" },+ { url = "https://files.pythonhosted.org/packages/25/13/44aa28d97f10e25247e8576b5f6b2795d4fa1a80acc88acc942c508d06f7/ast_serialize-0.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c9e763d70293d65ce1e1ea8c943140c68d0953f0268c7ee0998f2e07f77dd0", size = 1266458, upload-time = "2026-04-30T23:24:33.088Z" },+ { url = "https://files.pythonhosted.org/packages/d8/58/b3a8be3777cd3744324fd5cec0d80d37cd96fc7cbb0fb010e03dff1e870f/ast_serialize-0.3.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4388a1796c228f1ce5c391426f7d21a0003ad3b47f677dbeded9bd1a85c7209f", size = 1308700, upload-time = "2026-04-30T23:24:34.657Z" },+ { url = "https://files.pythonhosted.org/packages/13/03/f8312d6b57f5471a9dc7946f22b8798a1fc296d38c25766223aacadec42c/ast_serialize-0.3.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5283cdcc0c64c3d8b9b688dc6aaa012d9c0cf1380a7f774a6bae6a1c01b3205a", size = 1416724, upload-time = "2026-04-30T23:24:36.562Z" },+ { url = "https://files.pythonhosted.org/packages/50/5d/13fc3789a7abac00559da2e2e9f386db4612aa1f84fc53d09bf714c37545/ast_serialize-0.3.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f5ef88cc5842a5d7a6ac09dc0d5fc2c98f5d276c1f076f866d55047ce886785b", size = 1515441, upload-time = "2026-04-30T23:24:38.018Z" },+ { url = "https://files.pythonhosted.org/packages/eb/b9/7ab43fc7a23b1f970281093228f5f79bed6edeed7a3e672bde6d7a832a58/ast_serialize-0.3.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:cc14bf402bdc0978594ecce783793de2c7470cd4f5cd7eb286ca97ed8ff7cba9", size = 1510522, upload-time = "2026-04-30T23:24:39.798Z" },+ { url = "https://files.pythonhosted.org/packages/56/ec/d75fc2b788d319f1fad77c14156896f31afdfc68af85b505e5bdebcb9592/ast_serialize-0.3.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:11eae0cf1b7b3e0678133cc2daa974ea972caf02eb4b3aa062af6fa9acd52c57", size = 1460917, upload-time = "2026-04-30T23:24:41.305Z" },+ { url = "https://files.pythonhosted.org/packages/95/74/f99c81193a2725911e1911ae567ed27c2f2419332c7f3537366f9d238cac/ast_serialize-0.3.0-cp39-abi3-win32.whl", hash = "sha256:2db3dd99de5e6a5a11d7dda73de8750eb6e5baaf25245adf7bdcfe64b6108ae2", size = 1067804, upload-time = "2026-04-30T23:24:43.091Z" },+ { url = "https://files.pythonhosted.org/packages/16/81/76af00c47daa151e89f98ae21fbbcb2840aaa9f5766579c4da76a3c57188/ast_serialize-0.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:a2cd125adccf7969470621905d302750cd25951f22ea430d9a25b7be031e5549", size = 1105561, upload-time = "2026-04-30T23:24:44.578Z" },+ { url = "https://files.pythonhosted.org/packages/bd/46/d3ec57ad500f598d1554bd14ce4df615960549ab2844961bc4e1f5fbd174/ast_serialize-0.3.0-cp39-abi3-win_arm64.whl", hash = "sha256:0dd00da29985f15f50dc35728b7e1e7c84507bccfea1d9914738530f1c72238a", size = 1077165, upload-time = "2026-04-30T23:24:46.377Z" },+]+[[package]]name = "asyncpg"version = "0.31.0"@@ -1345,6 +1388,53 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" },]+[[package]]+name = "librt"+version = "0.10.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/39/cb/c1945e506893b5b8577fb45a60c80e3ffe4a82092a04a6f29b0b951d9a24/librt-0.10.0.tar.gz", hash = "sha256:1aba1e8aa4e3307a7be68a74149545fde7451964dc0235a8bec5704a17bdda42", size = 191799, upload-time = "2026-05-05T16:31:23.535Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e5/29/681a75c82f4cc90d29e4b257a3299b79fe13fe927a04c57b8109d70b6957/librt-0.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f0ede79d682e73f91c1b599a76d78b7464b9b5d213754cedb13372d9df36e596", size = 77299, upload-time = "2026-05-05T16:30:00.209Z" },+ { url = "https://files.pythonhosted.org/packages/62/24/0c7ca445a55d04be79cac19819437fd094782347fa116f6681844fa6143e/librt-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0ba0b131fdb336c8b9c948e397f4a7e649d0f783b529f07b647bf4961df392e", size = 79930, upload-time = "2026-05-05T16:30:01.555Z" },+ { url = "https://files.pythonhosted.org/packages/fe/1f/1e2b8f6443ef9e9a81e89486ca70e22f3684f93db003ce6eaefc3d0839b9/librt-0.10.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2728117da2afb96fb957768725ee43dc9a2d73b031e02da424b818a3cdd3a275", size = 246195, upload-time = "2026-05-05T16:30:03.261Z" },+ { url = "https://files.pythonhosted.org/packages/74/61/9dc9e03de0439ad84c1c240aac8b747f12c90cb797ea6042f7bdb8d3410f/librt-0.10.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:723ba80594c49cdf0584196fc430752262605dc9449902fc9bd3d9b79976cb77", size = 234951, upload-time = "2026-05-05T16:30:04.881Z" },+ { url = "https://files.pythonhosted.org/packages/55/f4/635223117d7590875bca441275065a3bf491203ad4208bd1cc3ffd90c5a1/librt-0.10.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7292edaaca294a61a978c53a3c7d6130d099b0dfbc8f0a65916cdc6b891b9852", size = 262768, upload-time = "2026-05-05T16:30:06.638Z" },+ { url = "https://files.pythonhosted.org/packages/e5/66/b04152d0cd8b6ca2b428a8bd3230343230c35ed304a932f35b5375f2f828/librt-0.10.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89fe9d539f2c10a1666633eeeac507ce95dd06d9ecc58de3c6390dba156a3d3a", size = 255075, upload-time = "2026-05-05T16:30:08.216Z" },+ { url = "https://files.pythonhosted.org/packages/35/1e/25bac4c7f2ca36f0e612cade186970683cf79153d96beccc3a11a9e19b97/librt-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4efa7b9587503fa5b67f40593302b9c8836d211d222ff9f7cafe67be5f8f0b10", size = 268559, upload-time = "2026-05-05T16:30:10.1Z" },+ { url = "https://files.pythonhosted.org/packages/18/54/4601faab35b6632a13200faa146ca62bfd111ffbe2568be430d65c89493a/librt-0.10.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:22dc982ef59df0136df36092ccbdbb570ced8aafb33e49585739b2f1de1c13b6", size = 261753, upload-time = "2026-05-05T16:30:11.912Z" },+ { url = "https://files.pythonhosted.org/packages/1b/cf/39f4023509e94fade8b074666fa3292db9cb6b34ea5dcbe7af53df9fca1d/librt-0.10.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6f2e5f3606253a84cea719c94a3bb1c54487b5d617d0254d46e0920d8a06be3f", size = 264055, upload-time = "2026-05-05T16:30:13.465Z" },+ { url = "https://files.pythonhosted.org/packages/8e/00/40247209fc46a8e308a91412d5206aedf8efb667ee89eb625820106a5c2f/librt-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:40884bfaa1e29f6b6a9be255007d8f359bfc9e61d68bdef8ed3158bfcbc95df9", size = 286190, upload-time = "2026-05-05T16:30:15.073Z" },+ { url = "https://files.pythonhosted.org/packages/d8/6e/5566beb94431a985abe1787af5ef86e087750172ff9d0bbf20f93e88132d/librt-0.10.0-cp313-cp313-win32.whl", hash = "sha256:3cd34cd8254eba756660bff6c2da91278248184301054fe3e4feb073bdd49b14", size = 62949, upload-time = "2026-05-05T16:30:16.503Z" },+ { url = "https://files.pythonhosted.org/packages/d0/c2/3ea3301d6c8dff51d39dbe8ed75db3dc92896947d4afb5eeadf821c1e67f/librt-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:7baac5313e2d8dce1386f97777a8d03ab28f5fe1e780b3b9ac2ee7544551fedc", size = 71152, upload-time = "2026-05-05T16:30:17.766Z" },+ { url = "https://files.pythonhosted.org/packages/3c/de/5d49cb92cadcbc77d3abc27b93fd6030ed8437487dde2eae38cab5e6704d/librt-0.10.0-cp313-cp313-win_arm64.whl", hash = "sha256:afc5b4406c8e2515698d922a5c7823a009312835ea58196671fff40e35cb8166", size = 61336, upload-time = "2026-05-05T16:30:19.021Z" },+ { url = "https://files.pythonhosted.org/packages/6a/64/7165e08108cc185a13a9c069f0685e6ef92e70e07fddf7edf5e7348c6316/librt-0.10.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:f09588a30e6a22ec624090d72a3ab1a6d4d5485c3ed739603e76aa3c16efa688", size = 76794, upload-time = "2026-05-05T16:30:20.392Z" },+ { url = "https://files.pythonhosted.org/packages/ae/ef/bf8613febf651b90c5222ee79dea5ae58d4cc2b544df69d3033424448934/librt-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:131ade118d12bd7a0adc4e655474a553f1b76cf78385868885944d21d51e45e0", size = 79662, upload-time = "2026-05-05T16:30:22.025Z" },+ { url = "https://files.pythonhosted.org/packages/b6/67/9eddd165c1d8397bdf99b38bf12b5a55b3def5035b49eedb49f2775d1430/librt-0.10.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8b9ab28e40d011c373a189eae900c916e66d6fbecf7983e9e4883089ee085ef", size = 242390, upload-time = "2026-05-05T16:30:23.51Z" },+ { url = "https://files.pythonhosted.org/packages/10/d1/d95da80334501866cd37004ab5d7483220d05862fab4b5405394f0264f0d/librt-0.10.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:67c39bb30da73bae1f293d1ed8bc2f8f6642649dd0928d3600aeff3041ac23d6", size = 232603, upload-time = "2026-05-05T16:30:25.198Z" },+ { url = "https://files.pythonhosted.org/packages/0c/fa/e6d64d28718bc1be4e1736fcb037ca1c4dfca927e7167df75a7d5215665e/librt-0.10.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c3273c6b774614f093c8927c2bf1b077d0fefde988fe98f46a333734e5597ab", size = 259187, upload-time = "2026-05-05T16:30:26.772Z" },+ { url = "https://files.pythonhosted.org/packages/72/3f/3fdb77e7f937dad59cfd76b720be7e7643400ec76b2da35befab8d66ba30/librt-0.10.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9dd7c1b86a4baa583ab5db977484b93a2c474e69e96ef3e9538387ea54229cb9", size = 251846, upload-time = "2026-05-05T16:30:28.56Z" },+ { url = "https://files.pythonhosted.org/packages/18/ca/f4d49133dd86a6f55d79eca30bf412fa722f511a9abe67f62f57aa64e66a/librt-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a77385c5a202e831149f7ad03be9e67cf80e957e52c614e83dcb822c95222eb8", size = 264936, upload-time = "2026-05-05T16:30:30.491Z" },+ { url = "https://files.pythonhosted.org/packages/de/66/a8df2fbadc1f6c1827a096d11c40175bd526133480bd3bc88ec64a03d257/librt-0.10.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c6a5eafa74b5655bad59886138ed68426f098a6beb8cb95a71f2cc3cd8bb33fe", size = 258699, upload-time = "2026-05-05T16:30:32.002Z" },+ { url = "https://files.pythonhosted.org/packages/bb/73/1e3c83613fe05451bb969e27b68a573d177f08d5f63533cc29fec0989658/librt-0.10.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1fc93d0439204c50ab4d1512611ce2c206f1b369b419f69c7c27c761561e3291", size = 259825, upload-time = "2026-05-05T16:30:35.077Z" },+ { url = "https://files.pythonhosted.org/packages/09/24/5e2f926ee9d3ef348d9339526d7062abb5c44d8419e3179528c01d78c102/librt-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:79e713c178bc7a744adfbee6b4619a288eecc0c914da2a9313a20255abe2f0cf", size = 282548, upload-time = "2026-05-05T16:30:36.639Z" },+ { url = "https://files.pythonhosted.org/packages/fc/7d/3e89ed6ad0162561fa8bef9df3195e24263104c955713cd0237d3711fad2/librt-0.10.0-cp314-cp314-win32.whl", hash = "sha256:2eba9d955a68c41d9f326be3da42f163ec3518b7ab20f1c826224e7bed71e0bf", size = 58970, upload-time = "2026-05-05T16:30:38.183Z" },+ { url = "https://files.pythonhosted.org/packages/76/25/579e731c94a7086a268bfa3e7a4945cd47836bebd3cbf3faeafd2e7eaef9/librt-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbfaf7f5145e9917f5d18bffa298eff6a19d74e7b8b11dabdca95785befe8dbf", size = 67260, upload-time = "2026-05-05T16:30:39.804Z" },+ { url = "https://files.pythonhosted.org/packages/6e/f8/235822b7ae0b2334f12ee18bcf2476d07924077a5efeea57dbe927704be2/librt-0.10.0-cp314-cp314-win_arm64.whl", hash = "sha256:8d6d385d1969849a6b1397114df22714b6ded917bada98668e3e974dc663477e", size = 57156, upload-time = "2026-05-05T16:30:41.412Z" },+ { url = "https://files.pythonhosted.org/packages/9f/e3/9b919cbf1e8eb770bf91bb7df28125e0f1daf4587169afefd95402636e9a/librt-0.10.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:6c3a82d3bd32631ef5c79922dfc028520c9ad840255979ab4d908271818039ee", size = 79150, upload-time = "2026-05-05T16:30:42.761Z" },+ { url = "https://files.pythonhosted.org/packages/6a/f5/72a944aa3bc3498169a168087eff58ca48b58bf1b704e59d091fd30739f3/librt-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d64cc66005dc324c9bb1fa3fc2841f529002f6eb15966d55e46d430f56955a6a", size = 82304, upload-time = "2026-05-05T16:30:44.082Z" },+ { url = "https://files.pythonhosted.org/packages/9c/e3/fcc290a33e295019759472dfa794d204e43504b276ac65eab7fd9da20ea3/librt-0.10.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9bb562cd28c88cd2c6a9a6c78f99dc39348d6b16c94adc25de0e574acf1176e9", size = 272556, upload-time = "2026-05-05T16:30:45.497Z" },+ { url = "https://files.pythonhosted.org/packages/fd/54/546975e4c997573885e7f040a05012f8838e06fb12b0c3c1fbb76254e9d7/librt-0.10.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:b809aa2854d019c28773b03605df22adc675ee4f3f4402d673581313e8906119", size = 256941, upload-time = "2026-05-05T16:30:47.059Z" },+ { url = "https://files.pythonhosted.org/packages/70/8c/f1d03401571b331653acddbd4e8cd955c06d945241dd08b25192fac0d04b/librt-0.10.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc15acabdd519bd4176fdadc2119e5e3093485d86f89138daf47e5b4cedb983a", size = 285855, upload-time = "2026-05-05T16:30:48.86Z" },+ { url = "https://files.pythonhosted.org/packages/0c/08/62cf80ff046c339faf56718b3a940244d4beb70f1c6407289b5830ec11e9/librt-0.10.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b1b2d835307d08ddadd94568e2369648ec9173bd3eea6d7f52a1abe717c81f98", size = 275321, upload-time = "2026-05-05T16:30:50.63Z" },+ { url = "https://files.pythonhosted.org/packages/d9/ea/da5918d4070362e9a4d2ee9cd34f9dc84902daad8fd4275f8504a727ff4e/librt-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d261c6a2f93335a5167887fb0223e8b98ffce20ee3fde242e8e58a37ece6d0e5", size = 293993, upload-time = "2026-05-05T16:30:52.577Z" },+ { url = "https://files.pythonhosted.org/packages/c9/8d/68b6086bed1fcdc314c640ea04e31e52d18052e08059fa595409d66a51a9/librt-0.10.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e2ffd44963f8e7f68995504d90f9881d64e94dc1d8e310039b9526108fc0c0f7", size = 284254, upload-time = "2026-05-05T16:30:55.086Z" },+ { url = "https://files.pythonhosted.org/packages/06/c8/b810f1d84ec34a5a7ed93d7b510ab04164d75fbdf23088d5c3fbe6b08357/librt-0.10.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f285f6455ed495791c4d8630e5af732960adea93cac4c893d15619f2eae53e8", size = 284925, upload-time = "2026-05-05T16:30:56.728Z" },+ { url = "https://files.pythonhosted.org/packages/5a/00/3c82d4158c5a2c62528b8fccce65a8c9ad700e480e86f9389387435089a5/librt-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f6034ff52e663d34c7b82ef2aa2f94ad7c1d939e2368e63b06844bc4d127d2e1", size = 307830, upload-time = "2026-05-05T16:30:58.377Z" },+ { url = "https://files.pythonhosted.org/packages/99/3a/9c635ac3e8a00383ff689161d3eac8a30b3b2ddc711b40471e6b8983ea29/librt-0.10.0-cp314-cp314t-win32.whl", hash = "sha256:657860fd877fba6a241ea088ef99f63ca819945d3c715265da670bad56c37ebe", size = 60147, upload-time = "2026-05-05T16:31:00.293Z" },+ { url = "https://files.pythonhosted.org/packages/dc/e8/6f65f3e565d4ac212cddddd552eacc8035ffdf941ca0ad6fe945a211d41f/librt-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:56ded2d66010203a0cb5af063b609e3f079531a0e5e576d618dece859fd2e1af", size = 68649, upload-time = "2026-05-05T16:31:01.778Z" },+ { url = "https://files.pythonhosted.org/packages/51/78/a0705a67cacd81e5fa01a5035b3adbdfbb43a7b8d4bd27e2b282ae61baf2/librt-0.10.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1ee63f30abf18ed4830fdbaf87b2b6f4bba1e198d46085c314edde4045e56715", size = 58247, upload-time = "2026-05-05T16:31:03.191Z" },+]+[[package]]name = "linkify-it-py"version = "2.1.0"@@ -1623,6 +1713,52 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" },]+[[package]]+name = "mypy"+version = "2.0.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "ast-serialize" },+ { name = "librt", marker = "platform_python_implementation != 'PyPy'" },+ { name = "mypy-extensions" },+ { name = "pathspec" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/cf/dc/7e6d49f04fca40b9dd5c752a51a432ffe67fb45200702bc9eee0cb4bbb26/mypy-2.0.0.tar.gz", hash = "sha256:1a9e3900ac5c40f1fe813506c7739da6e6f0eab2729067ebd94bfb0bbba53532", size = 3869036, upload-time = "2026-05-06T19:26:43.22Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e5/c6/996a1e535e5d0d597c3b1460fc962733091f885f312e749350eb2ac10965/mypy-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ef5f581b61240d1cc629b12f8df6565ed6ffac0d82ed745eef7833222ab50b9", size = 14737259, upload-time = "2026-05-06T19:20:23.081Z" },+ { url = "https://files.pythonhosted.org/packages/94/c5/0f9460e26b77f434bd53f47d1ce32a3cd4580c92a5331fa5dfc059f9421a/mypy-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:20e3470a165dbc249bdfbe8d1c5172727ef22688cffc279f8c3aa264ab9d4d9a", size = 13538377, upload-time = "2026-05-06T19:21:08.804Z" },+ { url = "https://files.pythonhosted.org/packages/b2/3e/8ea2f8dd1e5c9c279fb3c28193bdb850adf4d3d8172880abad829eced609/mypy-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:224ba142eee8b4d65d4db657cb1fc22abec30b135ded6ab297302ba1f62e505d", size = 13914264, upload-time = "2026-05-06T19:24:12.875Z" },+ { url = "https://files.pythonhosted.org/packages/be/ce/78bd3b8520f676acee9dab48ea71473e68f6d5cf14b59fbd800bea50a92b/mypy-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e879ad8a03908ff74d15e8a9b42bf049918e6798d52c011011f1873d0b5877e", size = 14926761, upload-time = "2026-05-06T19:20:12.846Z" },+ { url = "https://files.pythonhosted.org/packages/61/ef/b52fa340522da3d22e669117c3b83155c2660f7cdc035856958fbfffb224/mypy-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:65c5c15bcbd18d6fe927cc55c459597a3517d69cc3123f067be3b020010e115e", size = 15157014, upload-time = "2026-05-06T19:25:49.78Z" },+ { url = "https://files.pythonhosted.org/packages/7a/0c/dde7614250c6d017936c7aa3bb63b9b52c7cfd298d3f1be9be45f307870b/mypy-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:d1a068acd7c9fb77e9f8923f1556f2f49d6d7895821121b8d97fa5642b9c52f5", size = 11067049, upload-time = "2026-05-06T19:21:16.116Z" },+ { url = "https://files.pythonhosted.org/packages/27/ec/1d6af4830a94a285442db19caa02f160cc1a255e4f324eec5458e6c2bafb/mypy-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:ef9d96da1ddffbc21f27d3939319b6846d12393baa17c4d2f3e81e040e73ce2c", size = 9967903, upload-time = "2026-05-06T19:22:15.52Z" },+ { url = "https://files.pythonhosted.org/packages/ce/2c/6fefe954207860aed6eeb91776795e64a257d3ce0360862288984ce121f5/mypy-2.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c918c64e8ce36557851b0347f84eb12f1965d3a06813c36df253eb0c0afd1d82", size = 14729633, upload-time = "2026-05-06T19:24:53.383Z" },+ { url = "https://files.pythonhosted.org/packages/23/d6/d336f5b820af189eb0390cce21de62d264c0a4e64713dfbe81bfc4fc7739/mypy-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:301f1a8ccc7d79b542ee218b28bb49443a83e194eb3d10da63ff1649e5aa5d34", size = 13559524, upload-time = "2026-05-06T19:22:24.906Z" },+ { url = "https://files.pythonhosted.org/packages/af/a6/d7bb54fde1770f0484e5fbdbdce37a41e95ed0a1cd493ec60ead111e356c/mypy-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fdf4ef489d44ce350bac3fd699907834e551d4c934e9cc862ef201215ab1558d", size = 13936018, upload-time = "2026-05-06T19:25:02.992Z" },+ { url = "https://files.pythonhosted.org/packages/7d/ba/5be51316b91e6a6bf6e3a8adb3de500e7e1fb5bf9491743b8cbc81a34a2c/mypy-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9cde2d0989f912fc850890f727d0d76495e7a6c5bdd9912a1efdb64952b4398d", size = 14910712, upload-time = "2026-05-06T19:25:21.83Z" },+ { url = "https://files.pythonhosted.org/packages/b7/37/e2c8c3b373e20ebfb66e6c83a99027fd67df4ec43b08879f74e822d2dc4c/mypy-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cdf05693c231a14fe37dbfce192a3a1372c26a833af4a80f550547742952e719", size = 15141499, upload-time = "2026-05-06T19:20:50.924Z" },+ { url = "https://files.pythonhosted.org/packages/12/36/07756f933e00416d912e35878cfcf89a593a3350a885691c0bb85ae0226a/mypy-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:73aee2da33a2237e66cbe84a94780e53599847e86bb3aa7b93e405e8cd9905f2", size = 11240511, upload-time = "2026-05-06T19:21:32.39Z" },+ { url = "https://files.pythonhosted.org/packages/70/05/79ac1f20f2397353f3845f7b8bb5d8006cda7c8ef9092f04f9de3c6135f2/mypy-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:1f6dcd8f39971f41edab2728c877c4ac8b50ad3c387ff2770423b79a05d23910", size = 10149336, upload-time = "2026-05-06T19:22:08.383Z" },+ { url = "https://files.pythonhosted.org/packages/53/e0/0db84e0ebbad6e99e566c68e4b465784f2a2294f7719e8db9d509ef23087/mypy-2.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a04e980b9275c76159da66c6e1723c7798306f9802b31bdaf9358d0c84030ce8", size = 15797362, upload-time = "2026-05-06T19:22:00.835Z" },+ { url = "https://files.pythonhosted.org/packages/0a/a4/14cc0768164dd53bec48aa41a20270b18df9bf72aa5054278bf133608315/mypy-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:33f9cf4825469b2bc73c53ba55f6d9a9b4cdb60f9e6e228745581520f29b8771", size = 14635914, upload-time = "2026-05-06T19:23:43.675Z" },+ { url = "https://files.pythonhosted.org/packages/08/48/d866a3e23b4dc5974c77d9cf65a435bf22de01a84dd4620917950e233960/mypy-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191675c3c7dc2a5c7722a035a6909c277f14046c5e4e02aa5fbf65f8524f08ad", size = 15270866, upload-time = "2026-05-06T19:22:34.756Z" },+ { url = "https://files.pythonhosted.org/packages/71/eb/de9ef94958eb2078a6b908ceb247757dc384d3a238d3bd6ed7d81de5eaf8/mypy-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3d26c4321a3b06fc9f04c741e0733af693f82d823f8e64e47b2e63b7f19fa84", size = 16093131, upload-time = "2026-05-06T19:23:56.541Z" },+ { url = "https://files.pythonhosted.org/packages/ad/07/0ab2c1a9d26e90942612724cbd5788f16b7810c5dd39bfcf79286c6c4524/mypy-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bbcbc4d5917ca6ce12de70e051de7f533e3bf92d548b41a38a2232a6fe356525", size = 16330685, upload-time = "2026-05-06T19:21:42.037Z" },+ { url = "https://files.pythonhosted.org/packages/a6/8f/46f85d1371a5be642dad263828118ae1efd536d91d8bd2000c68acff3920/mypy-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:dbc6ba6d40572ae49268531565793a8f07eac7fc65ad76d482c9b4c8765b6043", size = 12752017, upload-time = "2026-05-06T19:22:44.002Z" },+ { url = "https://files.pythonhosted.org/packages/7a/e6/94ca48800cac19eb28a58188a768aaec0d16cac0f373915f073058ab0855/mypy-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:77926029dfcb7e1a3ecb0acb2ddbb24ca36be03f7d623e1759ad5376be8f6c01", size = 10527097, upload-time = "2026-05-06T19:20:58.973Z" },+ { url = "https://files.pythonhosted.org/packages/5c/14/fd0694aa594d6e9f9fd16ce821be2eff295197a273262ef56ddcc1388d68/mypy-2.0.0-py3-none-any.whl", hash = "sha256:8a92b2be3146b4fa1f062af7eb05574cbf3e6eb8e1f14704af1075423144e4e5", size = 2673434, upload-time = "2026-05-06T19:26:32.856Z" },+]++[[package]]+name = "mypy-extensions"+version = "1.1.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },+]+[[package]]name = "oauthlib"version = "3.3.1"@@ -1729,10 +1865,12 @@ dependencies = [[package.dev-dependencies]dev = [+ { name = "mypy" },{ name = "pytest" },{ name = "pytest-asyncio" },{ name = "pytest-cov" },{ name = "pytest-mock" },+ { name = "ruff" },][package.metadata]@@ -1780,10 +1918,12 @@ requires-dist = [[package.metadata.requires-dev]dev = [+ { name = "mypy", specifier = ">=2.0.0" },{ name = "pytest", specifier = ">=9.0.3" },{ name = "pytest-asyncio", specifier = ">=0.21.0" },{ name = "pytest-cov", specifier = ">=4.0.0" },{ name = "pytest-mock", specifier = ">=3.12.0" },+ { name = "ruff", specifier = ">=0.15.12" },][[package]]@@ -1851,6 +1991,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/52/96/5a770e5c461462575474468e5af931cff9de036e7c2b4fea23c1c58d2cbe/pathable-0.5.0-py3-none-any.whl", hash = "sha256:646e3d09491a6351a0c82632a09c02cdf70a252e73196b36d8a15ba0a114f0a6", size = 16867, upload-time = "2026-02-20T08:46:59.536Z" },]+[[package]]+name = "pathspec"+version = "1.1.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" },+]+[[package]]name = "platformdirs"version = "4.9.6"@@ -2545,6 +2694,31 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" },]+[[package]]+name = "ruff"+version = "0.15.12"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/99/43/3291f1cc9106f4c63bdce7a8d0df5047fe8422a75b091c16b5e9355e0b11/ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6", size = 4643852, upload-time = "2026-04-24T18:17:14.305Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c3/6e/e78ffb61d4686f3d96ba3df2c801161843746dcbcbb17a1e927d4829312b/ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c", size = 10640713, upload-time = "2026-04-24T18:17:22.841Z" },+ { url = "https://files.pythonhosted.org/packages/ae/08/a317bc231fb9e7b93e4ef3089501e51922ff88d6936ce5cf870c4fe55419/ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c", size = 11069267, upload-time = "2026-04-24T18:17:30.105Z" },+ { url = "https://files.pythonhosted.org/packages/aa/a4/f828e9718d3dce1f5f11c39c4f65afd32783c8b2aebb2e3d259e492c47bd/ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5", size = 10397182, upload-time = "2026-04-24T18:17:07.177Z" },+ { url = "https://files.pythonhosted.org/packages/71/e0/3310fc6d1b5e1fdea22bf3b1b807c7e187b581021b0d7d4514cccdb5fb71/ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002", size = 10758012, upload-time = "2026-04-24T18:16:55.759Z" },+ { url = "https://files.pythonhosted.org/packages/11/c1/a606911aee04c324ddaa883ae418f3569792fd3c4a10c50e0dd0a2311e1e/ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5", size = 10447479, upload-time = "2026-04-24T18:16:51.677Z" },+ { url = "https://files.pythonhosted.org/packages/9d/68/4201e8444f0894f21ab4aeeaee68aa4f10b51613514a20d80bd628d57e88/ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6", size = 11234040, upload-time = "2026-04-24T18:17:16.529Z" },+ { url = "https://files.pythonhosted.org/packages/34/ff/8a6d6cf4ccc23fd67060874e832c18919d1557a0611ebef03fdb01fff11e/ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33", size = 12087377, upload-time = "2026-04-24T18:17:04.944Z" },+ { url = "https://files.pythonhosted.org/packages/85/f6/c669cf73f5152f623d34e69866a46d5e6185816b19fcd5b6dd8a2d299922/ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847", size = 11367784, upload-time = "2026-04-24T18:17:25.409Z" },+ { url = "https://files.pythonhosted.org/packages/e8/39/c61d193b8a1daaa8977f7dea9e8d8ba866e02ea7b65d32f6861693aa4c12/ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0", size = 11344088, upload-time = "2026-04-24T18:17:12.258Z" },+ { url = "https://files.pythonhosted.org/packages/c2/8d/49afab3645e31e12c590acb6d3b5b69d7aab5b81926dbaf7461f9441f37a/ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339", size = 11271770, upload-time = "2026-04-24T18:17:02.457Z" },+ { url = "https://files.pythonhosted.org/packages/46/06/33f41fe94403e2b755481cdfb9b7ef3e4e0ed031c4581124658d935d52b4/ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5", size = 10719355, upload-time = "2026-04-24T18:17:27.648Z" },+ { url = "https://files.pythonhosted.org/packages/0d/59/18aa4e014debbf559670e4048e39260a85c7fcee84acfd761ac01e7b8d35/ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd", size = 10462758, upload-time = "2026-04-24T18:17:32.347Z" },+ { url = "https://files.pythonhosted.org/packages/25/e7/cc9f16fd0f3b5fddcbd7ec3d6ae30c8f3fde1047f32a4093a98d633c6570/ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b", size = 10953498, upload-time = "2026-04-24T18:17:20.674Z" },+ { url = "https://files.pythonhosted.org/packages/72/7a/a9ba7f98c7a575978698f4230c5e8cc54bbc761af34f560818f933dafa0c/ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e", size = 11447765, upload-time = "2026-04-24T18:17:09.755Z" },+ { url = "https://files.pythonhosted.org/packages/ea/f9/0ae446942c846b8266059ad8a30702a35afae55f5cdc54c5adf8d7afdc27/ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20", size = 10657277, upload-time = "2026-04-24T18:17:18.591Z" },+ { url = "https://files.pythonhosted.org/packages/33/f1/9614e03e1cdcbf9437570b5400ced8a720b5db22b28d8e0f1bda429f660d/ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d", size = 11837758, upload-time = "2026-04-24T18:17:00.113Z" },+ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" },+]+[[package]]name = "s3transfer"version = "0.16.0"