OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import asyncio
import datetime
import io
import json
import time
import uuid
from pathlib import Path
from typing import Any, NoReturn
import httpx
from config.settings import (
DOCLING_SERVE_VERIFY_SSL,
LANGFLOW_INGEST_CALLBACK_BATCH_SIZE,
LANGFLOW_INGEST_FLOW_ID,
LANGFLOW_URL_INGEST_FLOW_ID,
OPENRAG_BACKEND_ROUTER_ENABLE,
clients,
get_ingest_callback_url,
)
from services.document_index_writer import DocumentIndexContext
from utils.hash_utils import hash_id
from utils.langflow_utils import enable_mcp_none_for_project
from utils.logging_config import get_logger
logger = get_logger(__name__)
class LangflowFileService:
INGEST_OPENSEARCH_COMPONENT_ID = "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-WaE28"
URL_INGEST_OPENSEARCH_COMPONENT_ID = (
"OpenSearchVectorStoreComponentMultimodalMultiEmbedding-dihKf"
)
def __init__(
self,
flows_service=None,
docling_service=None,
document_index_writer=None,
ingest_token_service=None,
ingest_preview_service=None,
):
self.flow_id_ingest = LANGFLOW_INGEST_FLOW_ID
self.flows_service = flows_service
self.docling_service = docling_service
self.document_index_writer = document_index_writer
self.ingest_token_service = ingest_token_service
self.ingest_preview_service = ingest_preview_service
self.flow_id_url_ingest = LANGFLOW_URL_INGEST_FLOW_ID
self._embedding_dimension_cache: dict[str, int] = {}
_TRANSIENT_STATUS_CODES = {408, 429, 500, 502, 503, 504}
@classmethod
def _is_transient_status(cls, status_code: int) -> bool:
return status_code in cls._TRANSIENT_STATUS_CODES
@staticmethod
def _is_transient_request_error(error: Exception) -> bool:
return isinstance(
error,
(
httpx.TimeoutException,
httpx.NetworkError,
httpx.RequestError,
),
)
@staticmethod
def merge_ui_ingest_settings_into_tweaks(
tweaks: dict[str, Any] | None,
settings: dict[str, Any] | None,
) -> dict[str, Any]:
"""Merge UI ingest dict (camelCase) into Langflow run ``tweaks``.
- ``chunkSize`` / ``chunkOverlap`` / ``separator`` update the flow's
``Split Text`` node when any of those keys are present.
- ``embeddingModel`` is intentionally not mapped to a component tweak.
The embedding model should be supplied via
``run_ingestion_flow(..., selected_embedding_model=...)`` so Langflow
resolves it through the global variable override, without relying on
provider-specific component ids.
"""
final_tweaks = dict(tweaks) if tweaks else {}
from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, get_openrag_config
config = get_openrag_config()
# Build and merge Docling Serve tweaks
from services.docling_service import get_docling_preset_configs
preset_config = get_docling_preset_configs(
table_structure=config.knowledge.table_structure,
ocr=config.knowledge.ocr,
picture_descriptions=config.knowledge.picture_descriptions,
)
if "Docling Serve" not in final_tweaks:
final_tweaks["Docling Serve"] = {}
if "docling_serve_opts" not in final_tweaks["Docling Serve"]:
final_tweaks["Docling Serve"]["docling_serve_opts"] = preset_config
# Merge in default Split Text tweaks
if "Split Text" not in final_tweaks:
final_tweaks["Split Text"] = {}
if "chunk_size" not in final_tweaks["Split Text"]:
final_tweaks["Split Text"]["chunk_size"] = getattr(
config.knowledge, "chunk_size", DEFAULT_CHUNK_SIZE
)
if "chunk_overlap" not in final_tweaks["Split Text"]:
final_tweaks["Split Text"]["chunk_overlap"] = getattr(
config.knowledge, "chunk_overlap", DEFAULT_CHUNK_OVERLAP
)
if not settings:
return final_tweaks
if settings.get("chunkSize") or settings.get("chunkOverlap") or settings.get("separator"):
if settings.get("chunkSize"):
final_tweaks["Split Text"]["chunk_size"] = settings["chunkSize"]
if settings.get("chunkOverlap"):
final_tweaks["Split Text"]["chunk_overlap"] = settings["chunkOverlap"]
if settings.get("separator"):
final_tweaks["Split Text"]["separator"] = settings["separator"]
return final_tweaks
async def _detect_embedding_dimensions(
self,
embedding_model: str,
embedding_provider: str | None,
) -> int:
"""Generate one probe embedding so mapping dimensions match the provider."""
cache_key = f"{embedding_provider or ''}:{embedding_model}"
cached = self._embedding_dimension_cache.get(cache_key)
if cached:
return cached
# Through the gateway, like ingestion: the probe decides the index
# mapping's vector width, so it has to reach the same endpoint the real
# embedding call will. The direct client resolved credentials from
# process-global environment instead of from config.
from observability.tracing import trace_context
from services.llm_gateway import embeddings as gateway_embeddings
from services.llm_gateway import qualified_model_id
with trace_context(purpose="dimension_probe"):
response = await gateway_embeddings(
{
"model": qualified_model_id(embedding_provider, embedding_model),
"input": ["dimension probe"],
}
)
data = response.get("data") or []
if not data:
raise RuntimeError("Embedding provider returned no data for dimension probe")
first = data[0]
embedding = first["embedding"] if isinstance(first, dict) else first.embedding
dimensions = len(embedding)
if dimensions <= 0:
raise RuntimeError("Embedding provider returned an empty dimension probe")
self._embedding_dimension_cache[cache_key] = dimensions
return dimensions
async def _ensure_langflow_ingest_index(self, embedding_model: str | None) -> None:
"""Pre-create index mappings Langflow cannot manage with a DLS JWT."""
if clients.opensearch is None:
logger.debug("[LF] OpenSearch admin client unavailable; skipping ingest preflight")
return
try:
from config.embedding_constants import get_declared_default_embedding_model
from config.settings import get_index_name, get_openrag_config
from utils.embedding_fields import (
ensure_embedding_field_exists,
get_embedding_space_id,
)
from utils.embeddings import create_index_body
config = get_openrag_config()
index_name = get_index_name()
embedding_provider = config.knowledge.embedding_provider or "openai"
model_name = embedding_model or get_declared_default_embedding_model(embedding_provider)
embedding_dimensions = await self._detect_embedding_dimensions(
model_name,
embedding_provider,
)
embedding_space_id = get_embedding_space_id(
embedding_provider,
model_name,
)
if not await clients.opensearch.indices.exists(index=index_name):
await clients.opensearch.indices.create(
index=index_name,
body=await create_index_body(
model_name,
embedding_dimensions,
embedding_provider=embedding_provider,
embedding_space_id=embedding_space_id,
),
)
await ensure_embedding_field_exists(
clients.opensearch,
embedding_space_id,
index_name,
embedding_dimensions,
)
except Exception as e:
logger.warning(
"[LF] Failed to preconfigure OpenSearch index before Langflow ingest",
embedding_model=embedding_model,
error=str(e),
)
def _resolve_document_id(
self,
file_tuples: list[tuple[str, Any, str]] | None,
document_id: str | None,
connector_file_id: str | None = None,
) -> str:
# Bucket-style connectors (COS/Azure/S3) pass their raw, potentially
# non-ASCII and unbounded "bucket::key" as connector_file_id. Hash it
# into a stable ASCII document_id instead of using it verbatim — the
# raw value still travels downstream as connector_file_id, but
# document_id must stay safe for HTTP headers and OpenSearch's chunk
# _id (mirrors the content-hash document_id used by manual upload).
if connector_file_id:
return hash_id(io.BytesIO(connector_file_id.encode("utf-8")))
if document_id:
return document_id
if file_tuples and len(file_tuples[0]) > 1:
content: Any = file_tuples[0][1]
if isinstance(content, str):
content = content.encode("utf-8")
if isinstance(content, bytes):
return hash_id(io.BytesIO(content))
return str(uuid.uuid4())
def _configure_ingest_callback(
self,
*,
document_id: str,
mimetype: str,
file_size: int,
embedding_model: str,
filename: str | None = None,
owner: str | None,
owner_name: str | None,
owner_email: str | None,
connector_type: str | None,
source_url: str | None,
allowed_users: list[str] | None,
allowed_groups: list[str] | None,
allowed_principals: list[str] | None,
allowed_principal_labels: list[dict[str, Any]] | None = None,
parser: str | None = None,
chunk_size: int | None = None,
chunk_overlap: int | None = None,
connector_file_id: str | None = None,
) -> tuple[str | None, str | None]:
if self.ingest_token_service is None:
logger.warning(
"[LF] Backend-owned ingest delegation DISABLED: no ingest_token_service "
"wired. No OPENRAG_INGEST_* globals will be sent, so the Langflow "
"OpenSearch component will fall back to a direct write.",
document_id=document_id,
backend_router_enabled=OPENRAG_BACKEND_ROUTER_ENABLE,
)
return None, None
from config.settings import get_index_name, get_openrag_config
embedding_provider = (
getattr(get_openrag_config().knowledge, "embedding_provider", None) or "openai"
)
ingest_run_id = f"{document_id}-{uuid.uuid4().hex}"
context = DocumentIndexContext(
document_id=document_id,
filename=filename,
mimetype=mimetype,
embedding_model=embedding_model,
embedding_provider=embedding_provider,
owner=owner,
owner_name=owner_name,
owner_email=owner_email,
file_size=file_size,
connector_type=connector_type,
source_url=source_url,
allowed_users=allowed_users or [],
allowed_groups=allowed_groups or [],
allowed_principals=allowed_principals or [],
allowed_principal_labels=allowed_principal_labels or [],
ingest_run_id=ingest_run_id,
is_sample_data=connector_type == "openrag_docs",
index_name=get_index_name(),
parser=parser,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
connector_file_id=connector_file_id,
indexed_time=datetime.datetime.now(datetime.UTC).isoformat(),
)
token = self.ingest_token_service.create_token(context)
logger.info(
"[LF] Configured backend ingest callback",
document_id=document_id,
ingest_run_id=ingest_run_id,
index_name=context.index_name,
callback_url=get_ingest_callback_url(),
)
return token, ingest_run_id
def _ingest_callback_global_var_headers(
self,
*,
ingest_token: str | None,
ingest_run_id: str | None,
) -> dict[str, str]:
if not ingest_token or not ingest_run_id:
logger.warning(
"[LF] Ingest callback globals NOT attached to Langflow run "
"(missing token or run_id) — OpenSearch component will resolve "
"OPENRAG_INGEST_* to their placeholders and fall back to a direct "
"write instead of delegating to the backend.",
has_token=bool(ingest_token),
has_run_id=bool(ingest_run_id),
)
return {}
callback_url = get_ingest_callback_url()
logger.info(
"[LF] Ingest callback globals attached — delegating writes to backend",
ingest_run_id=ingest_run_id,
callback_url=callback_url,
batch_size=LANGFLOW_INGEST_CALLBACK_BATCH_SIZE,
)
return {
"X-Langflow-Global-Var-OPENRAG_INGEST_URL": callback_url,
"X-Langflow-Global-Var-OPENRAG_INGEST_TOKEN": ingest_token,
"X-Langflow-Global-Var-OPENRAG_INGEST_RUN_ID": ingest_run_id,
"X-Langflow-Global-Var-OPENRAG_INGEST_BATCH_SIZE": str(
LANGFLOW_INGEST_CALLBACK_BATCH_SIZE
),
}
async def _cleanup_failed_callback_ingest(
self,
*,
ingest_token: str | None,
ingest_run_id: str | None,
) -> None:
if self.ingest_token_service is not None and ingest_token:
self.ingest_token_service.revoke_token(ingest_token)
if self.document_index_writer is None or not ingest_run_id:
return
try:
await self.document_index_writer.delete_ingest_run(ingest_run_id)
except Exception as e:
logger.warning(
"[LF] Failed to clean up partial backend ingest run",
ingest_run_id=ingest_run_id,
error=str(e),
)
async def _raise_resolved_ingest_error(self, exc: BaseException) -> NoReturn:
"""Re-raise with a credential message when Langflow disconnects on bad API keys."""
from api.provider_validation import resolve_ingest_error_message
resolved = await resolve_ingest_error_message(exc)
if resolved != (str(exc) or "").strip():
raise Exception(resolved) from exc
raise exc
async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:
"""Upload a file using Langflow Files API v2: POST /api/v2/files.
Returns JSON with keys: id, name, path, size, provider.
"""
logger.debug("[LF] Upload (v2) -> /api/v2/files")
resp = await clients.langflow_request(
"POST",
"/api/v2/files",
files={"file": file_tuple},
headers={"Content-Type": None},
)
logger.debug(
"[LF] Upload response",
status_code=resp.status_code,
reason=resp.reason_phrase,
)
if resp.status_code >= 400:
logger.error(
"[LF] Upload failed",
status_code=resp.status_code,
reason=resp.reason_phrase,
body=resp.text,
)
resp.raise_for_status()
return resp.json()
async def delete_user_file(self, file_id: str) -> None:
"""Delete a file by id using v2: DELETE /api/v2/files/{id}."""
# NOTE: use v2 root, not /api/v1
logger.debug("[LF] Delete (v2) -> /api/v2/files/{id}", file_id=file_id)
resp = await clients.langflow_request("DELETE", f"/api/v2/files/{file_id}")
logger.debug(
"[LF] Delete response",
status_code=resp.status_code,
reason=resp.reason_phrase,
)
if resp.status_code >= 400:
logger.error(
"[LF] Delete failed",
status_code=resp.status_code,
reason=resp.reason_phrase,
body=resp.text[:500],
)
resp.raise_for_status()
async def run_ingestion_flow(
self,
file_paths: list[str],
file_tuples: list[tuple[str, str, str]],
jwt_token: str | None = None,
session_id: str | None = None,
tweaks: dict[str, Any] | None = None,
owner: str | None = None,
owner_name: str | None = None,
owner_email: str | None = None,
connector_type: str | None = None,
document_id: str | None = None,
connector_file_id: str | None = None,
source_url: str | None = None,
allowed_users: list[str] | None = None,
allowed_groups: list[str] | None = None,
allowed_principals: list[str] | None = None,
allowed_principal_labels: list[dict[str, Any]] | None = None,
selected_embedding_model: str | None = None,
docling_task_id: str | None = None,
original_filename: str | None = None,
original_mimetype: str | None = None,
) -> dict[str, Any]:
"""
Trigger the ingestion flow with provided file paths.
The flow must expose a File component path in input schema or accept files parameter.
"""
if not self.flow_id_ingest:
logger.error("[LF] LANGFLOW_INGEST_FLOW_ID is not configured")
raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")
payload: dict[str, Any] = {
"input_value": "Ingest files",
"input_type": "chat",
"output_type": "text", # Changed from "json" to "text"
}
if not tweaks:
tweaks = {}
from config.settings import get_openrag_config
config = get_openrag_config()
# Pass files via tweaks to File component (File-PSU37 from the flow)
if file_paths:
if "Docling Serve" not in tweaks:
tweaks["Docling Serve"] = {}
tweaks["Docling Serve"]["path"] = file_paths
if session_id:
payload["session_id"] = session_id
logger.debug(
"[LF] Run ingestion -> /run/%s | files=%s session_id=%s tweaks_keys=%s jwt_present=%s",
self.flow_id_ingest,
len(file_paths) if file_paths else 0,
session_id,
list(tweaks.keys()) if isinstance(tweaks, dict) else None,
bool(jwt_token),
)
# To compute the file size in bytes, use len() on the file content (which should be bytes)
file_size_bytes = len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0
# Avoid logging full payload to prevent leaking sensitive data (e.g., JWT)
# Extract file metadata if file_tuples is provided
filename = original_filename or (
str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""
)
mimetype = original_mimetype or (
str(file_tuples[0][2])
if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2
else ""
)
resolved_document_id = self._resolve_document_id(
file_tuples, document_id, connector_file_id
)
# Get the current embedding model and provider credentials from config
from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE
from services.langflow_llm_token_service import HOP_PURPOSE_INGEST
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
)
embedding_model = config.knowledge.embedding_model
if selected_embedding_model:
embedding_model = selected_embedding_model
split_tweaks = tweaks.get("Split Text", {}) if isinstance(tweaks, dict) else {}
chunk_size = split_tweaks.get(
"chunk_size", getattr(config.knowledge, "chunk_size", DEFAULT_CHUNK_SIZE)
)
chunk_overlap = split_tweaks.get(
"chunk_overlap", getattr(config.knowledge, "chunk_overlap", DEFAULT_CHUNK_OVERLAP)
)
headers = {
"X-Langflow-Global-Var-JWT": str(jwt_token or ""),
"X-Langflow-Global-Var-OWNER": owner or "",
"X-Langflow-Global-Var-OWNER_NAME": owner_name or "",
"X-Langflow-Global-Var-OWNER_EMAIL": owner_email or "",
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
"X-Langflow-Global-Var-MIMETYPE": mimetype,
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
**build_model_provider_headers(config, embedding_model=embedding_model),
"X-Langflow-Global-Var-DOCUMENT_ID": resolved_document_id,
"X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "",
"X-Langflow-Global-Var-DOCLING_TASK_ID": str(docling_task_id)
if docling_task_id
else "",
"X-Langflow-Global-Var-DOCLING_SERVE_VERIFY_SSL": str(DOCLING_SERVE_VERIFY_SSL).lower(),
}
# Serialize ACL lists as JSON strings for Langflow global vars
# (flows will parse these back into lists before indexing)
headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(allowed_users or [])
headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(allowed_groups or [])
headers["X-Langflow-Global-Var-ALLOWED_PRINCIPALS"] = json.dumps(allowed_principals or [])
headers["X-Langflow-Global-Var-ALLOWED_PRINCIPAL_LABELS"] = json.dumps(
allowed_principal_labels or []
)
ingest_token, ingest_run_id = self._configure_ingest_callback(
document_id=resolved_document_id,
filename=filename,
mimetype=mimetype,
file_size=file_size_bytes,
embedding_model=embedding_model,
owner=owner,
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
source_url=source_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,
allowed_principal_labels=allowed_principal_labels,
parser="Docling Serve 1.20.0",
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
connector_file_id=connector_file_id,
)
headers.update(
self._ingest_callback_global_var_headers(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
)
if tweaks:
payload["tweaks"] = tweaks
logger.debug("[LF] Tweaks configured", tweak_keys=list(tweaks.keys()))
# Add provider credentials as global variables for ingestion
await add_provider_credentials_to_headers(
headers,
config,
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=owner,
purpose=HOP_PURPOSE_INGEST,
)
if self.ingest_token_service is None:
await self._ensure_langflow_ingest_index(embedding_model)
start_time = time.time()
logger.info(
"[INGEST] Run started",
flow_id=self.flow_id_ingest,
filename=filename,
mimetype=mimetype,
)
try:
resp = await clients.langflow_request(
"POST",
f"/api/v1/run/{self.flow_id_ingest}",
json=payload,
headers=headers,
)
duration = round(time.time() - start_time, 2)
logger.info(
"[INGEST] Run complete",
status_code=resp.status_code,
reason=resp.reason_phrase,
duration_s=duration,
)
if resp.status_code >= 400:
logger.error(
"[LF] Run failed",
status_code=resp.status_code,
reason=resp.reason_phrase,
body=resp.text[:1000],
)
# Extract error message from Langflow response
error_message = f"Server error '{resp.status_code} {resp.reason_phrase}'"
try:
error_data = resp.json()
if isinstance(error_data, dict) and "detail" in error_data:
detail = error_data["detail"]
if isinstance(detail, str):
try:
detail_obj = json.loads(detail)
if isinstance(detail_obj, dict) and "message" in detail_obj:
error_message = detail_obj["message"]
else:
error_message = detail
except json.JSONDecodeError:
error_message = detail
elif isinstance(detail, dict) and "message" in detail:
error_message = detail["message"]
except Exception:
pass
raise Exception(error_message)
# Check if response is actually JSON before parsing
content_type = resp.headers.get("content-type", "")
if "application/json" not in content_type:
logger.error(
"[LF] Unexpected response content type from Langflow",
content_type=content_type,
status_code=resp.status_code,
body=resp.text[:1000],
)
raise ValueError(
f"Langflow returned {content_type} instead of JSON. "
f"This may indicate the ingestion flow failed or the endpoint is incorrect. "
f"Response preview: {resp.text[:500]}"
)
try:
resp_json = resp.json()
except Exception as e:
logger.error(
"[LF] Failed to parse run response as JSON",
body=resp.text[:1000],
error=str(e),
)
raise
return resp_json
except asyncio.CancelledError:
# Clean up partial ingestion before propagating cancellation
try:
await asyncio.shield(
self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
)
except Exception:
pass # Best-effort cleanup; don't block cancellation
raise
except Exception as e:
# Handle ordinary ingestion failures with cleanup
await self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
await self._raise_resolved_ingest_error(e)
async def run_url_ingestion_flow(
self,
docs_url: str,
crawl_depth: int,
jwt_token: str | None = None,
owner: str | None = None,
owner_name: str | None = None,
owner_email: str | None = None,
connector_type: str = "url",
prevent_outside: bool = True,
tweaks: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Run URL-based docs ingestion flow using Langflow global variable passthrough."""
if not docs_url:
raise ValueError("DEFAULT_DOCS_URL is not configured")
flow_id = await self._ensure_url_ingest_flow_id()
payload: dict[str, Any] = {
"input_value": docs_url,
"input_type": "chat",
"output_type": "text",
}
if not tweaks:
tweaks = {}
from config.settings import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, get_openrag_config
from services.langflow_llm_token_service import HOP_PURPOSE_INGEST
from utils.langflow_headers import (
add_provider_credentials_to_headers,
build_model_provider_headers,
)
config = get_openrag_config()
embedding_model = config.knowledge.embedding_model
resolved_document_id = hash_id(io.BytesIO(docs_url.encode("utf-8")))
split_tweaks = tweaks.get("Split Text", {}) if isinstance(tweaks, dict) else {}
default_chunk_size = getattr(config.knowledge, "chunk_size", DEFAULT_CHUNK_SIZE)
default_chunk_overlap = getattr(config.knowledge, "chunk_overlap", DEFAULT_CHUNK_OVERLAP)
chunk_size = split_tweaks.get("chunk_size", default_chunk_size)
chunk_overlap = split_tweaks.get("chunk_overlap", default_chunk_overlap)
headers = {
"X-Langflow-Global-Var-JWT": str(jwt_token or ""),
"X-Langflow-Global-Var-OWNER": owner or "",
"X-Langflow-Global-Var-OWNER_NAME": owner_name or "",
"X-Langflow-Global-Var-OWNER_EMAIL": owner_email or "",
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
**build_model_provider_headers(config, embedding_model=embedding_model),
"X-Langflow-Global-Var-DOCUMENT_ID": resolved_document_id,
"X-Langflow-Global-Var-SOURCE_URL": str(docs_url),
"X-Langflow-Global-Var-ALLOWED_USERS": json.dumps([]),
"X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps([]),
"X-Langflow-Global-Var-ALLOWED_PRINCIPALS": json.dumps([]),
"X-Langflow-Global-Var-DOCLING_TASK_ID": "",
"X-Langflow-Global-Var-MIMETYPE": "text/html",
"X-Langflow-Global-Var-FILESIZE": "0",
"X-Langflow-Global-Var-DOCLING_SERVE_VERIFY_SSL": str(DOCLING_SERVE_VERIFY_SSL).lower(),
}
ingest_token, ingest_run_id = self._configure_ingest_callback(
document_id=resolved_document_id,
mimetype="text/html",
file_size=0,
embedding_model=embedding_model,
owner=owner,
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
source_url=docs_url,
allowed_users=[],
allowed_groups=[],
allowed_principals=[],
allowed_principal_labels=[],
parser="URL Ingester",
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
headers.update(
self._ingest_callback_global_var_headers(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
)
if tweaks:
payload["tweaks"] = tweaks
await add_provider_credentials_to_headers(
headers,
config,
flows_service=self.flows_service,
jwt_token=jwt_token,
user_id=owner,
purpose=HOP_PURPOSE_INGEST,
)
if self.ingest_token_service is None:
await self._ensure_langflow_ingest_index(embedding_model)
logger.info(
"[LF] Running URL ingestion flow",
docs_url=docs_url,
crawl_depth=crawl_depth,
connector_type=connector_type,
embedding_model=embedding_model,
tweak_keys=list(tweaks.keys()),
)
try:
resp = await clients.langflow_request(
"POST",
f"/api/v1/run/{flow_id}",
json=payload,
headers=headers,
)
logger.info(
"[LF] URL ingestion flow response received",
status_code=resp.status_code,
flow_id=flow_id,
)
if resp.status_code >= 400:
logger.error(
"[LF] URL ingestion flow failed",
status_code=resp.status_code,
reason=resp.reason_phrase,
body=resp.text[:1000],
)
resp.raise_for_status()
content_type = resp.headers.get("content-type", "")
if "application/json" not in content_type:
logger.error(
"[LF] Unexpected URL ingestion response content type",
content_type=content_type,
status_code=resp.status_code,
body=resp.text[:1000],
)
raise ValueError(
f"Langflow returned {content_type} instead of JSON for URL ingestion. "
f"Response preview: {resp.text[:500]}"
)
return resp.json()
except asyncio.CancelledError:
# Clean up partial ingestion before propagating cancellation
try:
await asyncio.shield(
self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
)
except Exception:
pass # Best-effort cleanup; don't block cancellation
raise
except Exception as e:
# Handle ordinary ingestion failures with cleanup
await self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
await self._raise_resolved_ingest_error(e)
async def _ensure_url_ingest_flow_id(self) -> str:
"""Ensure URL ingest flow ID is valid; import flow if missing.
Retries once for transient Langflow failures so short outages do not
permanently block URL ingestion for the current process.
"""
configured_flow_id = self.flow_id_url_ingest
max_attempts = 2
last_error: Exception | None = None
from config.paths import get_flows_path
flow_file = Path(get_flows_path()) / "openrag_url_mcp.json"
if not flow_file.exists():
raise ValueError(
f"LANGFLOW_URL_INGEST_FLOW_ID is invalid and flow file was not found at {flow_file}"
)
with flow_file.open("r", encoding="utf-8") as f:
flow_payload = json.load(f)
for attempt in range(1, max_attempts + 1):
try:
if configured_flow_id:
check_resp = await clients.langflow_request(
"GET", f"/api/v1/flows/{configured_flow_id}"
)
if check_resp.status_code < 400:
try:
project_id = check_resp.json().get("folder_id")
if project_id:
await enable_mcp_none_for_project(project_id)
except Exception as err:
logger.warning(
f"[LF] Failed to configure MCP project auth on check: {err}"
)
return configured_flow_id
if check_resp.status_code != 404:
if self._is_transient_status(check_resp.status_code):
if attempt < max_attempts:
logger.warning(
"[LF] Transient URL ingest flow check failure, retrying once",
status_code=check_resp.status_code,
attempt=attempt,
max_attempts=max_attempts,
retry_in_seconds=1,
)
await asyncio.sleep(1)
continue
raise httpx.HTTPStatusError(
"URL ingest flow check failed",
request=check_resp.request,
response=check_resp,
)
logger.warning(
"[LF] URL ingest flow check returned non-404 error",
flow_id=configured_flow_id,
status_code=check_resp.status_code,
body_preview=check_resp.text[:300],
)
check_resp.raise_for_status()
logger.warning(
"[LF] URL ingest flow ID missing/invalid; importing flow JSON",
flow_file=str(flow_file),
previous_flow_id=configured_flow_id,
)
create_resp = await clients.langflow_request(
"POST", "/api/v1/flows/", json=flow_payload
)
if create_resp.status_code not in (200, 201):
if self._is_transient_status(create_resp.status_code):
if attempt < max_attempts:
logger.warning(
"[LF] Transient URL ingest flow import failure, retrying once",
status_code=create_resp.status_code,
attempt=attempt,
max_attempts=max_attempts,
retry_in_seconds=1,
)
await asyncio.sleep(1)
continue
raise httpx.HTTPStatusError(
"URL ingest flow import failed",
request=create_resp.request,
response=create_resp,
)
logger.error(
"[LF] Failed to import URL ingest flow",
status_code=create_resp.status_code,
body_preview=create_resp.text[:500],
)
create_resp.raise_for_status()
flow_data = create_resp.json()
imported_flow_id = flow_data.get("id")
project_id = flow_data.get("folder_id")
if project_id:
try:
await enable_mcp_none_for_project(project_id)
except Exception as err:
logger.warning(
f"[LF] Failed to configure MCP project auth after import: {err}"
)
if not imported_flow_id:
raise ValueError("Langflow flow import succeeded but no flow id was returned")
self.flow_id_url_ingest = imported_flow_id
logger.warning(
"[LF] Imported URL ingest flow for current runtime",
imported_flow_id=imported_flow_id,
note="Persist this in LANGFLOW_URL_INGEST_FLOW_ID to avoid re-importing on restart.",
)
return imported_flow_id
except httpx.RequestError as e:
last_error = e
if attempt == max_attempts:
raise
logger.warning(
"[LF] Transient request error during URL ingest auto-heal, retrying once",
attempt=attempt,
max_attempts=max_attempts,
retry_in_seconds=1,
error=str(e),
)
await asyncio.sleep(1)
except Exception as e:
last_error = e
raise
if last_error:
raise last_error
raise RuntimeError("Unable to validate/import URL ingest flow")
async def _cache_docling_preview_if_enabled(
self,
*,
preview_mode: bool,
upload_task_id: str | None,
preview_user_id: str | None,
docling_task_id: str,
document_id: str | None,
owner: str | None,
jwt_token: str | None,
file_path: str | None = None,
filename: str | None = None,
) -> None:
if not (
preview_mode and self.ingest_preview_service and upload_task_id and preview_user_id
):
return
try:
doc_json = await self.docling_service.fetch_task_result(
docling_task_id,
user_id=owner,
auth_header=jwt_token,
)
self.ingest_preview_service.store_docling_preview(
preview_user_id,
upload_task_id,
doc_json,
file_path=file_path,
document_id=document_id,
filename=filename,
)
except Exception as preview_error:
logger.warning(
"[LF] Failed to cache parse preview after Docling success",
extra={
"task_id": docling_task_id,
"upload_task_id": upload_task_id,
"error": str(preview_error),
},
)
async def submit_to_docling(
self,
filename: str,
content: bytes,
jwt_token: str | None = None,
owner: str | None = None,
*,
ocr: bool | None = None,
picture_descriptions: bool | None = None,
preview_mode: bool = False,
) -> str:
"""Upload a file to Docling Serve and return the task_id immediately.
Phase 1 of the two-phase ingestion model. The caller is responsible
for polling Docling (typically via DoclingPollingService) and only
invoking Langflow once Docling reports SUCCESS.
"""
if self.docling_service is None:
raise RuntimeError(
"DoclingService is not configured. Ensure DOCLING_SERVE_URL is set "
"and the service was injected correctly."
)
from services.docling_service import DoclingPayloadTooLarge
from utils.upload_limits import UploadTooLarge
try:
task_id = await self.docling_service.upload_to_docling_direct_async(
filename,
content,
user_id=owner,
auth_header=jwt_token,
ocr=ocr,
picture_descriptions=picture_descriptions,
preview_mode=preview_mode,
)
logger.debug(
"[LF] Docling submission accepted",
extra={"task_id": task_id, "filename": filename},
)
return task_id
except Exception as e:
logger.error(
"[LF] Docling submission failed",
extra={"error": str(e), "filename": filename},
)
if isinstance(e, (UploadTooLarge, DoclingPayloadTooLarge)):
raise
raise Exception(f"Docling upload failed: {str(e)}") from e
async def upload_and_ingest_file(
self,
file_tuple,
session_id: str | None = None,
tweaks: dict[str, Any] | None = None,
settings: dict[str, Any] | None = None,
jwt_token: str | None = None,
owner: str | None = None,
owner_name: str | None = None,
owner_email: str | None = None,
connector_type: str | None = None,
docling_polling_service: Any | None = None,
file_task: Any | None = None,
document_id: str | None = None,
connector_file_id: str | None = None,
source_url: str | None = None,
allowed_users: list[str] | None = None,
allowed_groups: list[str] | None = None,
allowed_principals: list[str] | None = None,
allowed_principal_labels: list[dict[str, Any]] | None = None,
original_filename: str | None = None,
original_mimetype: str | None = None,
preview_mode: bool = False,
upload_task_id: str | None = None,
preview_user_id: str | None = None,
) -> dict[str, Any]:
"""
Two-phase Docling upload + Langflow ingest operation.
Phase 1: submit the file to Docling, receive a task_id. If a
``docling_polling_service`` is provided, poll the backend for Docling
completion before invoking Langflow. This keeps Langflow execution
slots free during long Docling conversions.
Phase 2: trigger the Langflow ingestion flow once Docling has
succeeded. The task_id is forwarded so the flow's DoclingRemote
component fetches the already-completed result instead of re-uploading
or re-polling.
When ``docling_polling_service`` is None, falls back to the legacy
single-step behavior (Langflow polls Docling itself), preserving
backward compatibility.
Args:
file_tuple: (filename, content, content_type)
docling_polling_service: optional DoclingPollingService for the
two-phase flow. When None, Langflow handles polling.
file_task: optional FileTask for phase / status tracking.
"""
from models.tasks import DoclingPhaseStatus, IngestionPhase
logger.debug("[LF] Starting two-phase Docling+Langflow ingest")
filename, content, _ = file_tuple
ocr_override = settings.get("ocr") if isinstance(settings, dict) else None
pic_desc_override = (
settings.get("pictureDescriptions") if isinstance(settings, dict) else None
)
# ── Phase 1: submit to Docling ──────────────────────────────────
if file_task is not None:
file_task.phase = IngestionPhase.DOCLING
file_task.docling_status = DoclingPhaseStatus.PENDING
task_id = await self.submit_to_docling(
filename,
content,
owner=owner,
jwt_token=jwt_token,
ocr=ocr_override,
picture_descriptions=pic_desc_override,
preview_mode=preview_mode,
)
if file_task is not None:
file_task.docling_task_id = task_id
file_task.docling_status = DoclingPhaseStatus.PROCESSING
# ── Phase 1b: backend-side polling (optional) ───────────────────
if docling_polling_service is not None:
from config.settings import (
DOCLING_POLL_BACKOFF_FACTOR,
DOCLING_POLL_INTERVAL_SECONDS,
DOCLING_POLL_MAX_INTERVAL_SECONDS,
DOCLING_POLL_MAX_SECONDS,
DOCLING_POLL_TRANSIENT_RETRIES,
)
from services.docling_polling_service import PollOutcome
poll_result = await docling_polling_service.poll_until_ready(
task_id=task_id,
poll_interval=DOCLING_POLL_INTERVAL_SECONDS,
max_seconds=DOCLING_POLL_MAX_SECONDS,
max_interval=DOCLING_POLL_MAX_INTERVAL_SECONDS,
backoff_factor=DOCLING_POLL_BACKOFF_FACTOR,
transient_retry_budget=DOCLING_POLL_TRANSIENT_RETRIES,
user_id=owner,
auth_header=jwt_token,
)
if poll_result.outcome != PollOutcome.SUCCESS:
if file_task is not None:
if poll_result.outcome == PollOutcome.EXPIRED:
file_task.docling_status = DoclingPhaseStatus.EXPIRED
else:
file_task.docling_status = DoclingPhaseStatus.FAILED
logger.error(
"[LF] Docling polling did not reach SUCCESS; skipping Langflow",
extra={
"task_id": task_id,
"filename": filename,
"outcome": poll_result.outcome.value,
"detail": poll_result.detail,
"elapsed_seconds": round(poll_result.elapsed_seconds, 2),
},
)
from services.docling_service import password_protected_pdf_error
detail = poll_result.detail
if (
poll_result.outcome == PollOutcome.FAILED
and detail
and isinstance(content, bytes)
):
detail = password_protected_pdf_error(detail, content) or detail
raise Exception(
f"Docling conversion did not complete ({poll_result.outcome.value}): "
f"{detail or 'no detail provided'}"
)
if file_task is not None:
file_task.docling_status = DoclingPhaseStatus.SUCCESS
await self._cache_docling_preview_if_enabled(
preview_mode=preview_mode,
upload_task_id=upload_task_id,
preview_user_id=preview_user_id,
docling_task_id=task_id,
document_id=document_id,
owner=owner,
jwt_token=jwt_token,
file_path=file_task.file_path if file_task is not None else None,
filename=filename,
)
logger.info(
"[LF] Docling conversion ready; proceeding to Langflow",
extra={
"task_id": task_id,
"filename": filename,
"elapsed_seconds": round(poll_result.elapsed_seconds, 2),
},
)
# ── Phase 2: trigger Langflow ingestion ─────────────────────────
final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(tweaks, settings)
if settings:
logger.debug(
"[LF] Applying ingestion settings",
extra={"settings": settings, "tweaks": final_tweaks},
)
if file_task is not None:
file_task.phase = IngestionPhase.LANGFLOW
_raw_em = settings.get("embeddingModel") if isinstance(settings, dict) else None
selected_embedding = (
_raw_em.strip() if isinstance(_raw_em, str) and _raw_em.strip() else None
)
try:
total_start_time = time.time()
ingest_result = await self.run_ingestion_flow(
file_paths=[], # Files are not uploaded to Langflow FS
file_tuples=[file_tuple],
jwt_token=jwt_token,
session_id=session_id,
tweaks=final_tweaks,
owner=owner,
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
docling_task_id=task_id,
document_id=document_id,
connector_file_id=connector_file_id,
source_url=source_url,
selected_embedding_model=selected_embedding,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,
allowed_principal_labels=allowed_principal_labels,
original_filename=original_filename,
original_mimetype=original_mimetype,
)
total_duration = round(time.time() - total_start_time, 2)
logger.info(f"[LF] Ingestion completed successfully in {total_duration}s")
except Exception as e:
logger.error(
"[LF] Ingestion failed during combined operation",
extra={"error": str(e), "filename": filename},
)
# Docling Serve has no cancel endpoint; let any orphan task expire.
raise
if file_task is not None:
file_task.phase = IngestionPhase.COMPLETE
# Legacy path leaves docling_status at PROCESSING because the
# backend never observed Docling completion directly. Langflow
# returning success implies its DoclingRemote component consumed
# the task, so Docling succeeded — mark SUCCESS to keep status
# fields coherent. Idempotent for the polling path.
file_task.docling_status = DoclingPhaseStatus.SUCCESS
return {
"status": "success",
"docling_task_id": task_id,
"ingestion": ingest_result,
"message": f"File '{filename}' processed via Docling and ingested successfully",
}