Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: add preview-mode ingest backend with live index proof (OSS + SaaS) (#2051)

Wallgaua85ae050ab67
diff --git a/.env.example b/.env.example
index 28aac7b5a..2dcf6bd44 100644
--- a/.env.example
+++ b/.env.example
@@ -411,6 +411,12 @@ SESSION_SECRET=
# - "on_prem": Private Cloud
OPENRAG_RUN_MODE=oss
+# Feature flag for the preview-mode ingest backend (default: false = disabled).
+# When true, the frontend can request preview ingests that show progress and a
+# live index proof without persisting Docling layout JSON. AND-ed with the run
+# mode: preview is only ever available in the "oss" and "saas" run modes above.
+# OPENRAG_INGEST_PREVIEW_ENABLED=false
+
# Permission/identity cache backend. Only "memory" is currently wired.
# When CACHE_BACKEND=memory the RBAC permission cache and OAuth-subject
# cache are per-process, so OpenRAG must run with UVICORN_WORKERS=1 (and
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 4f04eb405..79b1377b8 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -28,6 +28,7 @@ from services.connector_access_service import (
set_connector_access_bulk,
)
from session_manager import User
+from utils.ingest_preview_flag import is_ingest_preview_enabled
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient
@@ -592,6 +593,8 @@ class ConnectorSyncBody(BaseModel):
# rather than failing. Set by the provider upload UI after the user confirms
# overwrite in the duplicate dialog.
replace_duplicates: bool = False
+ # When True (OSS/SaaS), run the ingest in preview mode (same as direct upload).
+ preview: bool = False
# When True (COS only), index chunks without an owner field so OpenSearch DLS
# makes them visible to all users in the instance. Temporary CIO mechanism;
# not a full ACL feature. Defaults to False (private).
@@ -925,6 +928,10 @@ async def connector_sync(
selected_files = [f.get("id") for f in selected_files_raw if f.get("id")]
file_infos = selected_files_raw
+ # Preview mode is opt-in from the connector upload UI (OSS/SaaS). It applies
+ # to user-initiated ingests (explicit file selection), not automated re-sync.
+ preview_mode = body.preview and is_ingest_preview_enabled()
+
try:
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
@@ -1052,6 +1059,7 @@ async def connector_sync(
file_infos=file_infos,
ingest_settings=body.settings,
replace_duplicates=body.replace_duplicates,
+ preview_mode=preview_mode,
shared=body.shared,
)
elif body.sync_all or body.bucket_filter:
@@ -1156,6 +1164,7 @@ async def connector_sync(
new_ids,
jwt_token=jwt_token,
ingest_settings=body.settings,
+ preview_mode=preview_mode,
shared=body.shared,
)
)
@@ -1168,6 +1177,7 @@ async def connector_sync(
jwt_token=jwt_token,
ingest_settings=body.settings,
replace_duplicates=True,
+ preview_mode=preview_mode,
shared=body.shared,
)
)
diff --git a/src/api/ingest_preview.py b/src/api/ingest_preview.py
new file mode 100644
index 000000000..4ad764481
--- /dev/null
+++ b/src/api/ingest_preview.py
@@ -0,0 +1,67 @@
+"""Ephemeral ingest preview endpoints for preview-mode ingest."""
+
+from typing import Annotated, Any
+
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from dependencies import (
+ get_ingest_preview_service,
+ get_session_manager,
+ get_task_service,
+ require_permission,
+)
+from session_manager import User
+from utils.ingest_preview_flag import is_ingest_preview_enabled
+
+
+def _preview_unavailable_response() -> JSONResponse:
+ return JSONResponse(
+ {"error": "Ingest preview is not available in this run mode"},
+ status_code=404,
+ )
+
+
+def _require_preview_task(task_service: Any, user: User, task_id: str):
+ """Return (upload_task, error_response). Exactly one of the tuple values is None."""
+ if not is_ingest_preview_enabled():
+ return None, _preview_unavailable_response()
+
+ upload_task = task_service.get_upload_task(user.user_id, task_id)
+ if upload_task is None:
+ return None, JSONResponse({"error": "Task not found"}, status_code=404)
+ if not upload_task.preview_mode:
+ return None, JSONResponse({"error": "Task is not a preview ingest"}, status_code=404)
+ return upload_task, None
+
+
+async def get_index_proof(
+ task_id: str,
+ preview_service: Annotated[Any, Depends(get_ingest_preview_service)],
+ task_service: Annotated[Any, Depends(get_task_service)],
+ session_manager: Annotated[Any, Depends(get_session_manager)],
+ user: Annotated[User, Depends(require_permission("knowledge:upload"))],
+ file: str | None = None,
+):
+ """Return indexed chunk metadata proving embeddings landed in OpenSearch.
+
+ ``file`` selects a specific file within a multi-file preview task.
+ """
+ upload_task, error = _require_preview_task(task_service, user, task_id)
+ if error is not None:
+ return error
+
+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)
+ proof = await preview_service.get_index_proof(
+ upload_task=upload_task,
+ task_id=task_id,
+ opensearch_client=opensearch_client,
+ file_path=file,
+ )
+
+ if proof.get("error") == "not_preview_task":
+ return JSONResponse({"error": "Task is not a preview ingest"}, status_code=404)
+ if proof.get("error") == "file_not_found":
+ return JSONResponse({"error": "File not found in preview task"}, status_code=404)
+
+ return JSONResponse({"task_id": task_id, **proof})
diff --git a/src/api/router.py b/src/api/router.py
index 81165fb87..8b76ba27a 100644
--- a/src/api/router.py
+++ b/src/api/router.py
@@ -17,6 +17,7 @@ from dependencies import (
get_task_service,
)
from session_manager import User
+from utils.ingest_preview_flag import is_ingest_preview_enabled
from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -29,6 +30,7 @@ async def upload_ingest_router(
tweaks_json: str | None = Form(None, alias="tweaks"),
replace_duplicates: str = Form("true"),
create_filter: str = Form("false"),
+ preview: str = Form("false"),
document_service=Depends(get_document_service),
langflow_file_service=Depends(get_langflow_file_service),
session_manager=Depends(get_session_manager),
@@ -42,9 +44,14 @@ async def upload_ingest_router(
- If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses Langflow upload-ingest via task service
"""
disable_ingest_with_langflow = get_openrag_config().knowledge.disable_ingest_with_langflow
+ requested_preview = preview.lower() == "true"
+ preview_mode = requested_preview and is_ingest_preview_enabled()
+ if requested_preview and not preview_mode:
+ logger.debug("Ingest preview ignored — not available in this run mode")
logger.debug(
"Router upload_ingest endpoint called",
disable_langflow_ingest=disable_ingest_with_langflow,
+ preview_mode=preview_mode,
)
if disable_ingest_with_langflow:
@@ -53,6 +60,7 @@ async def upload_ingest_router(
upload_files=file,
replace_duplicates=replace_duplicates.lower() == "true",
create_filter=create_filter.lower() == "true",
+ preview_mode=preview_mode,
session_manager=session_manager,
task_service=task_service,
user=user,
@@ -67,6 +75,7 @@ async def upload_ingest_router(
tweaks_json=tweaks_json,
replace_duplicates=replace_duplicates.lower() == "true",
create_filter=create_filter.lower() == "true",
+ preview_mode=preview_mode,
langflow_file_service=langflow_file_service,
session_manager=session_manager,
task_service=task_service,
@@ -78,6 +87,7 @@ async def _traditional_upload_ingest_task(
upload_files: list[UploadFile],
replace_duplicates: bool,
create_filter: bool,
+ preview_mode: bool,
session_manager,
task_service,
user: User,
@@ -142,6 +152,7 @@ async def _traditional_upload_ingest_task(
original_filenames=file_path_to_original_filename,
replace_duplicates=replace_duplicates,
settings=settings,
+ preview_mode=preview_mode,
)
return JSONResponse(
@@ -151,6 +162,7 @@ async def _traditional_upload_ingest_task(
"file_count": len(upload_files),
"create_filter": create_filter,
"filename": original_filenames[0] if len(original_filenames) == 1 else None,
+ "preview_mode": preview_mode,
},
status_code=202,
)
@@ -180,6 +192,7 @@ async def _langflow_upload_ingest_task(
tweaks_json,
replace_duplicates: bool,
create_filter: bool,
+ preview_mode: bool,
langflow_file_service,
session_manager,
task_service,
@@ -251,6 +264,7 @@ async def _langflow_upload_ingest_task(
tweaks=tweaks,
settings=settings,
replace_duplicates=replace_duplicates,
+ preview_mode=preview_mode,
)
return JSONResponse(
@@ -260,6 +274,7 @@ async def _langflow_upload_ingest_task(
"file_count": len(upload_files),
"create_filter": create_filter,
"filename": original_filenames[0] if len(original_filenames) == 1 else None,
+ "preview_mode": preview_mode,
},
status_code=202,
)
diff --git a/src/api/v1/documents.py b/src/api/v1/documents.py
index 07f1f5203..e58590e2f 100644
--- a/src/api/v1/documents.py
+++ b/src/api/v1/documents.py
@@ -69,6 +69,10 @@ async def ingest_endpoint(
tweaks_json=tweaks,
replace_duplicates=replace_duplicates,
create_filter=create_filter,
+ # Preview ingest is a frontend-only feature; the v1 SDK does not expose
+ # it. Pass an explicit value so the Form("false") default sentinel is not
+ # forwarded when this function is called directly (not via form parsing).
+ preview="false",
document_service=document_service,
langflow_file_service=langflow_file_service,
session_manager=session_manager,
diff --git a/src/app/container.py b/src/app/container.py
index e7a3609cb..a7a7918c5 100644
--- a/src/app/container.py
+++ b/src/app/container.py
@@ -25,6 +25,7 @@ from services.document_index_writer import DocumentIndexWriter
from services.document_service import DocumentService
from services.flows_service import FlowsService
from services.group_acl_service import GroupACLService
+from services.ingest_preview_service import IngestPreviewService
from services.knowledge_filter_service import KnowledgeFilterService
from services.langflow_file_service import LangflowFileService
from services.langflow_ingest_token_service import LangflowIngestTokenService
@@ -82,6 +83,7 @@ async def initialize_services():
models_service = ModelsService()
document_index_writer = DocumentIndexWriter()
langflow_ingest_token_service = LangflowIngestTokenService()
+ ingest_preview_service = IngestPreviewService()
document_service = DocumentService(
session_manager=session_manager,
models_service=models_service,
@@ -232,6 +234,7 @@ async def initialize_services():
"langflow_mcp_service": langflow_mcp_service,
"docling_service": clients.docling_service,
"docling_polling_service": docling_polling_service,
+ "ingest_preview_service": ingest_preview_service,
"rbac_service": rbac_service,
"workspace_config_service": workspace_config_service,
}
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index ee84c46b5..097a488bc 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -16,6 +16,7 @@ from api import (
documents,
files,
flows,
+ ingest_preview,
knowledge_filter,
langflow_files,
langflow_ingest,
@@ -74,6 +75,14 @@ def register_internal_routes(app: FastAPI):
app.add_api_route("/upload_options", upload.upload_options, methods=["GET"], tags=["internal"])
app.add_api_route("/upload_bucket", upload.upload_bucket, methods=["POST"], tags=["internal"])
+ # Ingest preview endpoint (index proof for preview-mode ingests)
+ app.add_api_route(
+ "/ingest/preview/{task_id}/index-proof",
+ ingest_preview.get_index_proof,
+ methods=["GET"],
+ tags=["internal"],
+ )
+
# Task endpoints
# Literal sub-paths must be registered before the parameterised /{task_id}
# so Starlette does not absorb "enhanced" as a task_id value.
diff --git a/src/config/settings.py b/src/config/settings.py
index b6c1fa0da..2363c31b2 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -290,6 +290,18 @@ def is_azure_blob_enabled() -> bool:
return raw in ("true", "1", "yes", "on")
+def is_ingest_preview_flag_enabled() -> bool:
+ """Raw opt-in flag for the preview-mode ingest backend.
+
+ Read per-call (like the other feature-flag accessors in this module) so
+ runtime/test overrides of ``OPENRAG_INGEST_PREVIEW_ENABLED`` take effect
+ without a restart. This is only the flag itself; run-mode gating is applied
+ by ``utils.ingest_preview_flag.is_ingest_preview_enabled()``.
+ """
+ raw = os.getenv("OPENRAG_INGEST_PREVIEW_ENABLED", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
def is_cloud_context() -> bool:
"""True when connector policy and SaaS settings guards should apply."""
from utils.run_mode_utils import is_run_mode_saas
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 3f3ddc04b..72d7eede5 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -471,7 +471,10 @@ class ConnectorService:
# Create custom task using TaskService
task_id = await self.task_service.create_custom_task(
- user_id, file_ids, processor, original_filenames=original_filenames
+ user_id,
+ file_ids,
+ processor,
+ original_filenames=original_filenames,
)
return task_id
@@ -485,6 +488,7 @@ class ConnectorService:
file_infos: list[dict[str, Any]] = None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
+ preview_mode: bool = False,
shared: bool = False,
) -> str:
"""
@@ -662,7 +666,11 @@ class ConnectorService:
}
task_id = await self.task_service.create_custom_task(
- user_id, expanded_file_ids, processor, original_filenames=original_filenames
+ user_id,
+ expanded_file_ids,
+ processor,
+ original_filenames=original_filenames,
+ preview_mode=preview_mode,
)
return task_id
diff --git a/src/dependencies.py b/src/dependencies.py
index 8f7a95af4..8c4334025 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -156,6 +156,10 @@ def get_langflow_file_service(services: dict = Depends(get_services)):
return services["langflow_file_service"]
+def get_ingest_preview_service(services: dict = Depends(get_services)):
+ return services["ingest_preview_service"]
+
+
def get_document_index_writer(services: dict = Depends(get_services)):
return services["document_index_writer"]
diff --git a/src/models/processors.py b/src/models/processors.py
index ef1d7dbe1..88a4255b7 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -723,6 +723,10 @@ class DocumentFileProcessor(TaskProcessor):
# Compute hash
file_hash = hash_id(item)
+ # Chunks are indexed with document_id=file_hash (see
+ # process_document_standard -> DocumentIndexContext), so record it on
+ # the file_task for preview-mode index proof lookups.
+ file_task.document_id = file_hash
# Get file size
try:
@@ -1154,6 +1158,7 @@ class ConnectorFileProcessor(TaskProcessor):
self.user_id, self.owner_name, self.owner_email, self.shared
)
)
+ file_task.document_id = document.id
result = await self.connector_service.langflow_service.upload_and_ingest_file(
file_tuple=file_tuple,
session_id=None,
@@ -1399,10 +1404,6 @@ class LangflowFileProcessor(TaskProcessor):
self.settings = settings
self.replace_duplicates = replace_duplicates
self.connector_type = connector_type
- # Backend-side Docling polling coordinator. Injected by TaskService
- # from the container; gating by ENABLE_BACKEND_DOCLING_POLLING happens
- # at construction time in app.container. When None, the legacy
- # single-call ingestion path is used.
self.docling_polling_service = docling_polling_service
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
@@ -1475,6 +1476,9 @@ class LangflowFileProcessor(TaskProcessor):
# Prepare metadata tweaks similar to API endpoint
final_tweaks = self.tweaks.copy() if self.tweaks else {}
+ file_hash = hash_id(item)
+ file_task.document_id = file_hash
+
# Build settings with fresh OCR/pictureDescriptions from live
# config so retries pick up configuration changes.
config = get_openrag_config()
@@ -1498,6 +1502,7 @@ class LangflowFileProcessor(TaskProcessor):
connector_type=self.connector_type,
docling_polling_service=self.docling_polling_service,
file_task=file_task,
+ document_id=file_hash,
original_filename=original_filename,
original_mimetype=original_mimetype,
)
@@ -1507,8 +1512,10 @@ class LangflowFileProcessor(TaskProcessor):
# landed in OpenSearch before declaring success. We key off the
# filename — the identifier this path already uses for dedup and
# delete (see check_filename_exists / delete_document_by_filename
- # above) — because Langflow assigns its own document_id here, so
- # hash_id(item) is not stored as document_id.
+ # above). The document_id (hash_id(item) == content hash) is now
+ # threaded through to Langflow so preview-mode index proof can look
+ # chunks up by document_id, but verification stays filename-based to
+ # match this path's existing dedup/delete semantics.
#
# wait_for_visibility polls on an empty result so the just-written
# chunks become visible within OpenSearch's near-real-time refresh
diff --git a/src/models/tasks.py b/src/models/tasks.py
index 7dde4bee5..bfd9133bd 100644
--- a/src/models/tasks.py
+++ b/src/models/tasks.py
@@ -58,6 +58,7 @@ class FileTask:
docling_task_id: str | None = None
docling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDING
phase: IngestionPhase = IngestionPhase.DOCLING
+ document_id: str | None = None
@property
def duration_seconds(self) -> float:
@@ -81,6 +82,7 @@ class UploadTask:
processor: TaskProcessor | None = field(default=None, repr=False)
background_task: asyncio.Task[None] | None = field(default=None, repr=False)
temp_file_paths: list[str] | None = field(default=None, repr=False)
+ preview_mode: bool = False
_sequence_number: int = field(init=False, repr=False)
def __post_init__(self):
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index 4a2f16ff8..f2a47c9ba 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -380,10 +380,10 @@ class DoclingService:
if payload.get("status") == "failure" or payload.get("errors"):
raise DoclingServeError(f"Docling processing failed: {_format_docling_error(payload)}")
- doc_content = payload.get("document", {}).get("json_content")
- if doc_content is None:
+ document = payload.get("document") or {}
+ if document.get("json_content") is None:
raise DoclingServeError("docling-serve response missing document.json_content")
- return doc_content
+ return document["json_content"]
async def _poll_result(
self,
@@ -418,7 +418,8 @@ class DoclingService:
result_json = result_response.json()
# Extract the json_content which matches the old convert_file/bytes return
- doc_content = result_json.get("document", {}).get("json_content")
+ document = result_json.get("document") or {}
+ doc_content = document.get("json_content")
if doc_content is None:
raise DoclingServeError("docling-serve response missing document.json_content")
diff --git a/src/services/ingest_preview_service.py b/src/services/ingest_preview_service.py
new file mode 100644
index 000000000..d63ecf54a
--- /dev/null
+++ b/src/services/ingest_preview_service.py
@@ -0,0 +1,186 @@
+"""Index proof helpers for preview-mode ingest (no Docling document cache)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from config.settings import get_index_name
+from models.tasks import IngestionPhase
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+TEXT_PREVIEW_MAX_LENGTH = 240
+INDEX_PROOF_MAX_CHUNKS = 200
+
+
+def _chunk_sequence(chunk_id: str | None) -> int:
+ """Numeric suffix from chunk ``_id`` (last underscore-delimited segment)."""
+ if not chunk_id:
+ return 0
+ suffix = chunk_id.rsplit("_", 1)[-1]
+ try:
+ return int(suffix)
+ except ValueError:
+ return 0
+
+
+def _sort_hits(hits: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ return sorted(
+ hits,
+ key=lambda hit: (
+ (hit.get("_source") or {}).get("page") or 0,
+ _chunk_sequence(hit.get("_id")),
+ ),
+ )
+
+
+def _extract_hit_total(hits_section: dict[str, Any], fallback: int) -> int:
+ total = hits_section.get("total")
+ if isinstance(total, dict):
+ value = total.get("value")
+ return int(value) if value is not None else fallback
+ if isinstance(total, int):
+ return total
+ return fallback
+
+
+class IngestPreviewService:
+ """Stateless helper for preview-mode index proof queries."""
+
+ async def get_index_proof(
+ self,
+ *,
+ upload_task: Any,
+ task_id: str,
+ opensearch_client: Any,
+ file_path: str | None = None,
+ ) -> dict[str, Any]:
+ if not getattr(upload_task, "preview_mode", False):
+ return {
+ "ready": False,
+ "error": "not_preview_task",
+ "phase": IngestionPhase.DOCLING.value,
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": None,
+ }
+
+ if file_path is not None:
+ file_task = upload_task.file_tasks.get(file_path)
+ if file_task is None:
+ return {
+ "ready": False,
+ "error": "file_not_found",
+ "phase": IngestionPhase.DOCLING.value,
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": None,
+ }
+ else:
+ file_task = next(iter(upload_task.file_tasks.values()), None)
+ phase = file_task.phase.value if file_task is not None else IngestionPhase.DOCLING.value
+
+ document_id = file_task.document_id if file_task is not None else None
+
+ if file_task is None or file_task.phase != IngestionPhase.COMPLETE:
+ return {
+ "ready": False,
+ "phase": phase,
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": document_id,
+ }
+
+ if opensearch_client is None:
+ return {
+ "ready": False,
+ "phase": phase,
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": document_id,
+ "error": "opensearch_unavailable",
+ }
+
+ try:
+ response = await opensearch_client.search(
+ index=get_index_name(),
+ body={
+ "size": INDEX_PROOF_MAX_CHUNKS,
+ "query": {"term": {"document_id": document_id}},
+ "sort": [{"page": "asc"}],
+ "_source": {
+ "includes": [
+ "text",
+ "page",
+ "embedding_model",
+ "embedding_dimensions",
+ ]
+ },
+ },
+ )
+ except Exception as exc:
+ logger.warning(
+ "Failed to query index proof chunks",
+ task_id=task_id,
+ document_id=document_id,
+ error=str(exc),
+ )
+ return {
+ "ready": False,
+ "phase": phase,
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": document_id,
+ "error": "search_failed",
+ }
+
+ hits_section = response.get("hits", {})
+ hits = _sort_hits(hits_section.get("hits", []))
+ chunk_count = _extract_hit_total(hits_section, len(hits))
+ chunks_returned = len(hits)
+ chunks_truncated = chunk_count > chunks_returned
+ chunks = []
+ embedding_model = None
+ embedding_dimensions = None
+
+ for hit in hits:
+ source = hit.get("_source") or {}
+ text = source.get("text") or ""
+ if embedding_model is None and source.get("embedding_model"):
+ embedding_model = source["embedding_model"]
+ if embedding_dimensions is None and source.get("embedding_dimensions"):
+ embedding_dimensions = source["embedding_dimensions"]
+ preview_text = text.strip()
+ if len(preview_text) > TEXT_PREVIEW_MAX_LENGTH:
+ preview_text = preview_text[:TEXT_PREVIEW_MAX_LENGTH] + "…"
+ chunks.append(
+ {
+ "chunk_id": hit.get("_id"),
+ "page": source.get("page"),
+ "text_preview": preview_text,
+ "char_count": len(text),
+ }
+ )
+
+ return {
+ "ready": chunk_count > 0,
+ "phase": phase,
+ "chunk_count": chunk_count,
+ "chunks_returned": chunks_returned,
+ "chunks_truncated": chunks_truncated,
+ "embedding_model": embedding_model,
+ "embedding_dimensions": embedding_dimensions,
+ "chunks": chunks,
+ "document_id": document_id,
+ }
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 824cbb4a0..78e7e7002 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -253,6 +253,7 @@ class TaskService:
original_filenames: dict | None = None,
replace_duplicates: bool = False,
settings: dict | None = None,
+ preview_mode: bool = False,
) -> str:
"""Create a new upload task for bulk file processing"""
# Use default DocumentFileProcessor with user context
@@ -276,6 +277,7 @@ class TaskService:
processor,
original_filenames=original_filenames,
temp_file_paths=file_paths,
+ preview_mode=preview_mode,
)
async def create_langflow_upload_task(
@@ -295,6 +297,7 @@ class TaskService:
connector_type: str = "local",
existing_task_id: str = None,
temp_file_paths: list | None = None,
+ preview_mode: bool = False,
) -> str:
"""Create a new upload task for Langflow file processing with upload and ingest"""
# Use LangflowFileProcessor with user context
@@ -321,6 +324,7 @@ class TaskService:
original_filenames,
existing_task_id=existing_task_id,
temp_file_paths=temp_file_paths if temp_file_paths is not None else file_paths,
+ preview_mode=preview_mode,
)
async def create_langflow_url_upload_task(
@@ -366,6 +370,7 @@ class TaskService:
original_filenames: dict | None = None,
existing_task_id: str = None,
temp_file_paths: list | None = None,
+ preview_mode: bool = False,
) -> str:
"""Create a new task with custom processor for any type of items"""
import os
@@ -400,6 +405,7 @@ class TaskService:
task_id=task_id,
total_files=len(items),
file_tasks=file_tasks,
+ preview_mode=preview_mode,
)
upload_task.processor = processor
if store_user_id not in self.task_store:
@@ -754,6 +760,10 @@ class TaskService:
return self.task_store[candidate_user_id][task_id]
return None
+ def get_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
+ """Public accessor for upload task lookup."""
+ return self._resolve_upload_task(user_id, task_id)
+
def _resolve_upload_task_store(
self, user_id: str, task_id: str
) -> tuple[str, UploadTask] | None:
@@ -1216,7 +1226,11 @@ class TaskService:
file_statuses = {}
for file_path, file_task in upload_task.file_tasks.items():
- if file_task.status != TaskStatus.COMPLETED:
+ # The task panel only surfaces in-flight/failed files to keep
+ # the list tidy. Preview-mode tasks instead need every file
+ # (including completed ones) so the live preview carousel can
+ # still enumerate them and render their cached Docling layout.
+ if file_task.status != TaskStatus.COMPLETED or upload_task.preview_mode:
entry = self._serialize_file_task(file_task)
if file_task.status == TaskStatus.FAILED:
metadata = self._infer_failure_metadata(file_task)
diff --git a/src/utils/ingest_preview_flag.py b/src/utils/ingest_preview_flag.py
new file mode 100644
index 000000000..a6db0f6d2
--- /dev/null
+++ b/src/utils/ingest_preview_flag.py
@@ -0,0 +1,21 @@
+"""Feature-flag gating for the preview-mode ingest backend."""
+
+from config.settings import is_ingest_preview_flag_enabled
+from utils.run_mode_utils import is_run_mode_oss, is_run_mode_saas
+
+
+def is_ingest_preview_enabled() -> bool:
+ """Whether the preview-mode ingest backend is active.
+
+ Gated behind an explicit opt-in feature flag so merging to main does not
+ change behavior for existing installs. The flag defaults to ``false`` --
+ set ``OPENRAG_INGEST_PREVIEW_ENABLED=true`` to turn it on. The run-mode
+ check is AND-ed with the flag: preview is only ever available in OSS and
+ SaaS run modes (never on_prem), and even there only when the flag is set.
+
+ The raw env read lives in ``config.settings`` (the single place that owns
+ ``os.environ`` parsing); this module only layers the run-mode gate on top.
+ """
+ if not is_ingest_preview_flag_enabled():
+ return False
+ return is_run_mode_oss() or is_run_mode_saas()
diff --git a/src/utils/run_mode_utils.py b/src/utils/run_mode_utils.py
index a2dbd2314..f462ab04d 100644
--- a/src/utils/run_mode_utils.py
+++ b/src/utils/run_mode_utils.py
@@ -1,4 +1,5 @@
"""Utilities for determining the OpenRAG application run mode."""
+
import os
RUN_MODE_OSS: str = "oss"
diff --git a/tests/unit/api/test_connector_sync_response.py b/tests/unit/api/test_connector_sync_response.py
index 0148d9d1c..c8c9ff64d 100644
--- a/tests/unit/api/test_connector_sync_response.py
+++ b/tests/unit/api/test_connector_sync_response.py
@@ -105,3 +105,70 @@ async def test_connector_sync_reports_one_connection_with_multiple_active(monkey
service.sync_specific_files.assert_awaited_once()
args = service.sync_specific_files.await_args.args
assert args[0] == "conn-1" # connection_id of first connector
+
+
+def _preview_service_with_working_connection():
+ """Service mock whose single active connection authenticates and syncs."""
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[_make_connection("c1")])
+ service.get_connector = AsyncMock(return_value=connector)
+ service.sync_specific_files = AsyncMock(return_value="task-preview")
+ return service
+
+
+async def test_connector_sync_passes_preview_mode_to_sync_specific_files(monkeypatch):
+ """body.preview=True threads preview_mode=True into sync_specific_files (OSS/SaaS)."""
+ from api import connectors as connectors_api
+ from api import documents as documents_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "is_ingest_preview_enabled", lambda: True)
+ monkeypatch.setattr(documents_api, "_ensure_index_exists", AsyncMock())
+
+ service = _preview_service_with_working_connection()
+
+ response = await connectors_api.connector_sync(
+ "google_drive",
+ connectors_api.ConnectorSyncBody(selected_files=["file-a"], preview=True),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ assert service.sync_specific_files.await_args.kwargs["preview_mode"] is True
+
+
+async def test_connector_sync_ignores_preview_when_disabled(monkeypatch):
+ """body.preview=True is ignored (preview_mode=False) when the run mode disables preview."""
+ from api import connectors as connectors_api
+ from api import documents as documents_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "is_ingest_preview_enabled", lambda: False)
+ monkeypatch.setattr(documents_api, "_ensure_index_exists", AsyncMock())
+
+ service = _preview_service_with_working_connection()
+
+ response = await connectors_api.connector_sync(
+ "google_drive",
+ connectors_api.ConnectorSyncBody(selected_files=["file-a"], preview=True),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ assert service.sync_specific_files.await_args.kwargs["preview_mode"] is False
diff --git a/tests/unit/api/test_ingest_preview.py b/tests/unit/api/test_ingest_preview.py
new file mode 100644
index 000000000..a4d64f471
--- /dev/null
+++ b/tests/unit/api/test_ingest_preview.py
@@ -0,0 +1,151 @@
+"""API tests for ingest preview endpoints."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi.responses import JSONResponse
+
+from api.ingest_preview import get_index_proof
+from session_manager import User
+
+
[email protected](autouse=True)
+def enable_ingest_preview():
+ with patch("api.ingest_preview.is_ingest_preview_enabled", return_value=True):
+ yield
+
+
+def user():
+ return User(user_id="user-1", email="[email protected]", name="User", jwt_token="Bearer tok")
+
+
+def preview_service():
+ service = MagicMock()
+ service.get_index_proof = AsyncMock(
+ return_value={
+ "ready": True,
+ "phase": "complete",
+ "chunk_count": 2,
+ "embedding_model": "text-embedding-3-small",
+ "embedding_dimensions": 1536,
+ "chunks": [
+ {"chunk_id": "hash-1_0", "page": 1, "text_preview": "Hello", "char_count": 5}
+ ],
+ }
+ )
+ return service
+
+
+def task_service():
+ upload_task = MagicMock()
+ upload_task.preview_mode = True
+ service = MagicMock()
+ service.get_upload_task.return_value = upload_task
+ return service
+
+
+async def test_get_index_proof_uses_validated_upload_task_without_refetch(
+ user, preview_service, task_service
+):
+ """Regression: index proof must use the task validated by _require_preview_task."""
+ upload_task = task_service.get_upload_task.return_value
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client.return_value = AsyncMock()
+
+ await get_index_proof(
+ task_id="task-1",
+ preview_service=preview_service,
+ task_service=task_service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ task_service.get_upload_task.assert_called_once_with("user-1", "task-1")
+ call_kwargs = preview_service.get_index_proof.await_args.kwargs
+ assert call_kwargs["upload_task"] is upload_task
+ assert "task_service" not in call_kwargs
+ assert "user_id" not in call_kwargs
+
+
+async def test_get_index_proof_returns_chunk_metadata(user, preview_service, task_service):
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client.return_value = AsyncMock()
+
+ response = await get_index_proof(
+ task_id="task-1",
+ preview_service=preview_service,
+ task_service=task_service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ assert response.status_code == 200
+ preview_service.get_index_proof.assert_awaited_once()
+ call_kwargs = preview_service.get_index_proof.await_args.kwargs
+ assert call_kwargs["upload_task"] is task_service.get_upload_task.return_value
+ assert call_kwargs["task_id"] == "task-1"
+
+
+async def test_get_index_proof_task_not_found(user, preview_service):
+ task_service = MagicMock()
+ task_service.get_upload_task.return_value = None
+
+ response = await get_index_proof(
+ task_id="missing",
+ preview_service=preview_service,
+ task_service=task_service,
+ session_manager=MagicMock(),
+ user=user,
+ )
+
+ assert response.status_code == 404
+ preview_service.get_index_proof.assert_not_called()
+
+
+async def test_get_index_proof_returns_file_not_found(user, preview_service, task_service):
+ preview_service.get_index_proof = AsyncMock(
+ return_value={
+ "ready": False,
+ "error": "file_not_found",
+ "phase": "docling",
+ "chunk_count": 0,
+ "chunks": [],
+ "chunks_returned": 0,
+ "chunks_truncated": False,
+ "document_id": None,
+ }
+ )
+
+ response = await get_index_proof(
+ task_id="task-1",
+ preview_service=preview_service,
+ task_service=task_service,
+ session_manager=MagicMock(),
+ user=user,
+ file="/tmp/missing.pdf",
+ )
+
+ assert response.status_code == 404
+
+
+async def test_get_index_proof_unavailable_when_disabled(user, preview_service, task_service):
+ with patch("api.ingest_preview.is_ingest_preview_enabled", return_value=False):
+ response = await get_index_proof(
+ task_id="task-1",
+ preview_service=preview_service,
+ task_service=task_service,
+ session_manager=MagicMock(),
+ user=user,
+ )
+
+ assert isinstance(response, JSONResponse)
+ assert response.status_code == 404
+ preview_service.get_index_proof.assert_not_called()
diff --git a/tests/unit/api/test_v1_documents_ingest_preview.py b/tests/unit/api/test_v1_documents_ingest_preview.py
new file mode 100644
index 000000000..d097d262c
--- /dev/null
+++ b/tests/unit/api/test_v1_documents_ingest_preview.py
@@ -0,0 +1,38 @@
+"""Regression test: the public v1 ingest endpoint calls upload_ingest_router
+directly (not via FastAPI form parsing), so it must forward an explicit string
+for every Form-defaulted param. Previously ``preview`` was omitted, leaking the
+``Form("false")`` sentinel into ``preview.lower()`` and raising
+``AttributeError: 'Form' object has no attribute 'lower'`` (HTTP 500).
+"""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from api.v1.documents import ingest_endpoint
+from session_manager import User
+
+
+async def test_v1_ingest_forwards_preview_as_string():
+ user = User(user_id="user-1", email="[email protected]", name="User", jwt_token="Bearer tok")
+
+ with patch("api.v1.documents.upload_ingest_router", new=AsyncMock()) as mock_router:
+ await ingest_endpoint(
+ file=[MagicMock()],
+ session_id=None,
+ settings=None,
+ tweaks=None,
+ replace_duplicates="true",
+ create_filter="false",
+ document_service=MagicMock(),
+ langflow_file_service=MagicMock(),
+ session_manager=MagicMock(),
+ task_service=MagicMock(),
+ user=user,
+ )
+
+ call_kwargs = mock_router.await_args.kwargs
+ # Must be a real string so upload_ingest_router's preview.lower() works.
+ assert isinstance(call_kwargs["preview"], str)
+ assert call_kwargs["preview"] == "false"
diff --git a/tests/unit/config/test_disable_ingest_setting.py b/tests/unit/config/test_disable_ingest_setting.py
index c6ff0e571..7e45509b1 100644
--- a/tests/unit/config/test_disable_ingest_setting.py
+++ b/tests/unit/config/test_disable_ingest_setting.py
@@ -100,6 +100,7 @@ async def test_traditional_upload_ingest_task(monkeypatch):
upload_files=[mock_file],
replace_duplicates=True,
create_filter=False,
+ preview_mode=False,
session_manager=mock_session_manager,
task_service=mock_task_service,
user=mock_user,
@@ -162,6 +163,7 @@ async def test_langflow_upload_ingest_task(monkeypatch):
tweaks_json=None,
replace_duplicates=True,
create_filter=False,
+ preview_mode=False,
langflow_file_service=mock_langflow_file_service,
session_manager=mock_session_manager,
task_service=mock_task_service,
@@ -224,6 +226,7 @@ async def test_traditional_upload_ingest_mime_fallback(monkeypatch):
upload_files=[mock_file],
replace_duplicates=True,
create_filter=False,
+ preview_mode=False,
session_manager=mock_session_manager,
task_service=mock_task_service,
user=mock_user,
diff --git a/tests/unit/services/test_ingest_preview_service.py b/tests/unit/services/test_ingest_preview_service.py
new file mode 100644
index 000000000..2aeb6745b
--- /dev/null
+++ b/tests/unit/services/test_ingest_preview_service.py
@@ -0,0 +1,273 @@
+"""Tests for preview-mode index proof helpers."""
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from models.tasks import FileTask, IngestionPhase, TaskStatus, UploadTask
+from services.ingest_preview_service import (
+ IngestPreviewService,
+ _chunk_sequence,
+ _extract_hit_total,
+ _sort_hits,
+)
+
+
+def test_chunk_sequence_parses_numeric_suffix():
+ assert _chunk_sequence("hash-abc_10") == 10
+ assert _chunk_sequence("hash-abc_2") == 2
+ assert _chunk_sequence("bad-id") == 0
+
+
+def test_sort_hits_orders_by_page_then_chunk_sequence():
+ hits = [
+ {"_id": "doc_10", "_source": {"page": 1}},
+ {"_id": "doc_2", "_source": {"page": 1}},
+ {"_id": "doc_0", "_source": {"page": 2}},
+ ]
+ sorted_hits = _sort_hits(hits)
+ assert [hit["_id"] for hit in sorted_hits] == ["doc_2", "doc_10", "doc_0"]
+
+
+def test_extract_hit_total_prefers_total_value():
+ assert _extract_hit_total({"total": {"value": 42}, "hits": []}, 0) == 42
+ assert _extract_hit_total({"total": 7, "hits": []}, 0) == 7
+ assert _extract_hit_total({"hits": []}, 3) == 3
+
+
+def test_extract_hit_total_falls_back_when_value_is_null():
+ # OpenSearch may return {"total": {"value": null}}; must not raise TypeError.
+ assert _extract_hit_total({"total": {"value": None}, "hits": []}, 5) == 5
+
+
+async def test_get_index_proof_selects_file_by_path():
+ service = IngestPreviewService()
+
+ file_a = FileTask(file_path="/tmp/a.pdf", filename="a.pdf")
+ file_a.phase = IngestionPhase.LANGFLOW
+ file_b = FileTask(file_path="/tmp/b.pdf", filename="b.pdf", document_id="hash-b")
+ file_b.phase = IngestionPhase.COMPLETE
+ file_b.status = TaskStatus.COMPLETED
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=2,
+ file_tasks={"/tmp/a.pdf": file_a, "/tmp/b.pdf": file_b},
+ preview_mode=True,
+ )
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {"hits": {"hits": [], "total": {"value": 0}}}
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=opensearch_client,
+ file_path="/tmp/b.pdf",
+ )
+
+ assert proof["phase"] == "complete"
+ assert proof["document_id"] == "hash-b"
+ searched_body = opensearch_client.search.await_args.kwargs["body"]
+ assert searched_body["query"]["term"]["document_id"] == "hash-b"
+ assert "_id" not in str(searched_body.get("sort", []))
+
+
+async def test_get_index_proof_rejects_non_preview_task():
+ service = IngestPreviewService()
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=0,
+ file_tasks={},
+ preview_mode=False,
+ )
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=AsyncMock(),
+ )
+
+ assert proof["ready"] is False
+ assert proof["error"] == "not_preview_task"
+
+
+async def test_get_index_proof_not_ready_while_ingesting():
+ service = IngestPreviewService()
+ file_task = FileTask(
+ file_path="/tmp/sample.pdf",
+ filename="sample.pdf",
+ document_id="hash-sample",
+ )
+ file_task.phase = IngestionPhase.LANGFLOW
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"/tmp/sample.pdf": file_task},
+ preview_mode=True,
+ )
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=AsyncMock(),
+ )
+
+ assert proof["ready"] is False
+ assert proof["phase"] == "langflow"
+ assert proof["chunk_count"] == 0
+
+
+async def test_get_index_proof_returns_chunks_when_indexed():
+ service = IngestPreviewService()
+
+ file_task = FileTask(
+ file_path="/tmp/sample.pdf",
+ filename="sample.pdf",
+ document_id="hash-abc",
+ )
+ file_task.phase = IngestionPhase.COMPLETE
+ file_task.status = TaskStatus.COMPLETED
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"/tmp/sample.pdf": file_task},
+ preview_mode=True,
+ )
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {
+ "hits": {
+ "hits": [
+ {
+ "_id": "hash-abc_10",
+ "_source": {
+ "text": "Later chunk",
+ "page": 1,
+ "embedding_model": "text-embedding-3-small",
+ "embedding_dimensions": 1536,
+ },
+ },
+ {
+ "_id": "hash-abc_2",
+ "_source": {
+ "text": "Earlier chunk",
+ "page": 1,
+ "embedding_model": "text-embedding-3-small",
+ "embedding_dimensions": 1536,
+ },
+ },
+ ],
+ "total": {"value": 250},
+ }
+ }
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=opensearch_client,
+ )
+
+ assert proof["ready"] is True
+ assert proof["chunk_count"] == 250
+ assert proof["chunks_returned"] == 2
+ assert proof["chunks_truncated"] is True
+ assert proof["embedding_model"] == "text-embedding-3-small"
+ assert proof["embedding_dimensions"] == 1536
+ assert len(proof["chunks"]) == 2
+ assert proof["chunks"][0]["chunk_id"] == "hash-abc_2"
+ assert proof["chunks"][1]["chunk_id"] == "hash-abc_10"
+ assert proof["chunks"][0]["char_count"] == len("Earlier chunk")
+
+
+async def test_get_index_proof_returns_file_not_found_for_unknown_path():
+ service = IngestPreviewService()
+ file_task = FileTask(
+ file_path="/tmp/sample.pdf",
+ filename="sample.pdf",
+ document_id="hash-sample",
+ )
+ file_task.phase = IngestionPhase.COMPLETE
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"/tmp/sample.pdf": file_task},
+ preview_mode=True,
+ )
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=AsyncMock(),
+ file_path="/tmp/missing.pdf",
+ )
+
+ assert proof["ready"] is False
+ assert proof["error"] == "file_not_found"
+
+
+async def test_get_index_proof_opensearch_unavailable():
+ service = IngestPreviewService()
+ file_task = FileTask(
+ file_path="/tmp/sample.pdf",
+ filename="sample.pdf",
+ document_id="hash-abc",
+ )
+ file_task.phase = IngestionPhase.COMPLETE
+ file_task.status = TaskStatus.COMPLETED
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"/tmp/sample.pdf": file_task},
+ preview_mode=True,
+ )
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=None,
+ )
+
+ assert proof["ready"] is False
+ assert proof["error"] == "opensearch_unavailable"
+ assert proof["phase"] == "complete"
+ assert proof["chunk_count"] == 0
+ assert proof["document_id"] == "hash-abc"
+
+
+async def test_get_index_proof_search_failure():
+ service = IngestPreviewService()
+ file_task = FileTask(
+ file_path="/tmp/sample.pdf",
+ filename="sample.pdf",
+ document_id="hash-abc",
+ )
+ file_task.phase = IngestionPhase.COMPLETE
+ file_task.status = TaskStatus.COMPLETED
+ upload_task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"/tmp/sample.pdf": file_task},
+ preview_mode=True,
+ )
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search.side_effect = RuntimeError("opensearch down")
+
+ proof = await service.get_index_proof(
+ upload_task=upload_task,
+ task_id="task-1",
+ opensearch_client=opensearch_client,
+ )
+
+ assert proof["ready"] is False
+ assert proof["error"] == "search_failed"
+ assert proof["phase"] == "complete"
+ assert proof["chunk_count"] == 0
+ assert proof["document_id"] == "hash-abc"
diff --git a/tests/unit/test_langflow_processor_preview_document_id.py b/tests/unit/test_langflow_processor_preview_document_id.py
new file mode 100644
index 000000000..8c3a86cbc
--- /dev/null
+++ b/tests/unit/test_langflow_processor_preview_document_id.py
@@ -0,0 +1,43 @@
+"""Regression tests for preview index-proof document_id threading."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from models.processors import LangflowFileProcessor
+from models.tasks import FileTask, TaskStatus, UploadTask
+from utils.hash_utils import hash_id
+
+
+async def test_langflow_processor_threads_document_id_for_index_proof(tmp_path):
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=AsyncMock())
+
+ langflow_file_service = MagicMock()
+ langflow_file_service.upload_and_ingest_file = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+
+ processor = LangflowFileProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ owner_user_id="user-1",
+ jwt_token="Bearer user-token",
+ )
+ processor.check_filename_exists = AsyncMock(side_effect=[False, True])
+
+ item = tmp_path / "report.pdf"
+ item.write_bytes(b"%PDF-1.4 dummy")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "My Report.pdf"
+ upload_task = UploadTask(task_id="task-1", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ expected_id = hash_id(str(item))
+ assert file_task.status == TaskStatus.COMPLETED
+ assert file_task.document_id == expected_id
+ call_kwargs = langflow_file_service.upload_and_ingest_file.await_args.kwargs
+ assert call_kwargs["document_id"] == expected_id
diff --git a/tests/unit/test_router_preview_upload.py b/tests/unit/test_router_preview_upload.py
new file mode 100644
index 000000000..506f693aa
--- /dev/null
+++ b/tests/unit/test_router_preview_upload.py
@@ -0,0 +1,98 @@
+"""Tests that preview=true is threaded through the upload router."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi import UploadFile
+
+from api.router import _langflow_upload_ingest_task, upload_ingest_router
+from session_manager import User
+
+
+async def test_langflow_upload_passes_preview_mode_to_task_service():
+ mock_file = MagicMock(spec=UploadFile)
+ mock_file.filename = "sample.pdf"
+ mock_file.content_type = "application/pdf"
+ mock_file.read = AsyncMock(return_value=b"%PDF-sample")
+
+ mock_task_service = MagicMock()
+ mock_task_service.create_langflow_upload_task = AsyncMock(return_value="task-preview-1")
+
+ user = User(user_id="user-1", email="[email protected]", name="User", jwt_token="Bearer tok")
+
+ mock_temp_file = MagicMock()
+ mock_temp_file.name = "/tmp/sample.pdf"
+
+ with (
+ patch("api.router.tempfile.NamedTemporaryFile", return_value=mock_temp_file),
+ patch("api.router.open", create=True),
+ patch("utils.file_utils.safe_unlink"),
+ patch("api.router.is_ingest_preview_enabled", return_value=True),
+ ):
+ response = await _langflow_upload_ingest_task(
+ upload_files=[mock_file],
+ session_id=None,
+ settings_json=None,
+ tweaks_json=None,
+ replace_duplicates=True,
+ create_filter=False,
+ preview_mode=True,
+ langflow_file_service=MagicMock(),
+ session_manager=MagicMock(),
+ task_service=mock_task_service,
+ user=user,
+ )
+
+ assert response.status_code == 202
+ call_kwargs = mock_task_service.create_langflow_upload_task.await_args.kwargs
+ assert call_kwargs["preview_mode"] is True
+
+ import json
+
+ body = json.loads(response.body.decode())
+ assert body["preview_mode"] is True
+
+
+async def test_upload_ingest_router_ignores_preview_when_disabled():
+ mock_file = MagicMock(spec=UploadFile)
+ mock_file.filename = "sample.pdf"
+ mock_file.content_type = "application/pdf"
+ mock_file.read = AsyncMock(return_value=b"%PDF-sample")
+
+ mock_task_service = MagicMock()
+ mock_task_service.create_langflow_upload_task = AsyncMock(return_value="task-1")
+
+ user = User(user_id="user-1", email="[email protected]", name="User", jwt_token="Bearer tok")
+
+ mock_temp_file = MagicMock()
+ mock_temp_file.name = "/tmp/sample.pdf"
+
+ with (
+ patch("api.router.get_openrag_config") as mock_cfg,
+ patch("api.router.tempfile.NamedTemporaryFile", return_value=mock_temp_file),
+ patch("api.router.open", create=True),
+ patch("utils.file_utils.safe_unlink"),
+ patch("api.router.is_ingest_preview_enabled", return_value=False),
+ ):
+ mock_cfg.return_value.knowledge.disable_ingest_with_langflow = False
+
+ response = await upload_ingest_router(
+ file=[mock_file],
+ session_id=None,
+ settings_json=None,
+ tweaks_json=None,
+ preview="true",
+ replace_duplicates="true",
+ create_filter="false",
+ langflow_file_service=MagicMock(),
+ session_manager=MagicMock(),
+ task_service=mock_task_service,
+ document_service=MagicMock(),
+ user=user,
+ )
+
+ assert response.status_code == 202
+ call_kwargs = mock_task_service.create_langflow_upload_task.await_args.kwargs
+ assert call_kwargs["preview_mode"] is False
diff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.py
index 9a0711015..5ae68739a 100644
--- a/tests/unit/test_task_service.py
+++ b/tests/unit/test_task_service.py
@@ -281,3 +281,46 @@ async def test_concurrent_mixed_counter_updates(task_service):
assert task.failed_files == 7
assert task.successful_files == 13
assert task.processed_files == task.successful_files + task.failed_files
+
+
+def test_enhanced_drops_completed_files_for_normal_tasks(task_service):
+ """Task panel keeps its list tidy: completed files are not surfaced."""
+ task = UploadTask(
+ task_id="normal_task",
+ total_files=2,
+ file_tasks={
+ "fileA": FileTask(file_path="fileA", status=TaskStatus.COMPLETED),
+ "fileB": FileTask(file_path="fileB", status=TaskStatus.RUNNING),
+ },
+ status=TaskStatus.RUNNING,
+ )
+ task_service.task_store["user1"] = {"normal_task": task}
+
+ tasks = task_service.get_all_tasks2("user1")
+
+ files = tasks[0]["files"]
+ assert "fileA" not in files # completed dropped
+ assert "fileB" in files
+
+
+def test_enhanced_keeps_completed_files_for_preview_tasks(task_service):
+ """Preview tasks must retain completed files so the live preview carousel
+ can still enumerate them and render their cached Docling layout."""
+ task = UploadTask(
+ task_id="preview_task",
+ total_files=2,
+ file_tasks={
+ "fileA": FileTask(file_path="fileA", status=TaskStatus.COMPLETED),
+ "fileB": FileTask(file_path="fileB", status=TaskStatus.RUNNING),
+ },
+ status=TaskStatus.RUNNING,
+ preview_mode=True,
+ )
+ task_service.task_store["user1"] = {"preview_task": task}
+
+ tasks = task_service.get_all_tasks2("user1")
+
+ files = tasks[0]["files"]
+ assert "fileA" in files # completed retained for preview
+ assert files["fileA"]["status"] == "completed"
+ assert "fileB" in files
diff --git a/tests/unit/utils/test_ingest_preview_flag.py b/tests/unit/utils/test_ingest_preview_flag.py
new file mode 100644
index 000000000..46233f59c
--- /dev/null
+++ b/tests/unit/utils/test_ingest_preview_flag.py
@@ -0,0 +1,28 @@
+"""Tests for the ingest-preview feature flag gating."""
+
+import pytest
+
+from utils.ingest_preview_flag import is_ingest_preview_enabled
+
+
[email protected]("run_mode", ["oss", "saas", "on_prem", ""])
+def test_disabled_by_default(monkeypatch, run_mode):
+ """Without the opt-in flag, preview is off in every run mode."""
+ monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)
+ monkeypatch.delenv("OPENRAG_INGEST_PREVIEW_ENABLED", raising=False)
+ assert is_ingest_preview_enabled() is False
+
+
[email protected]("flag", ["true", "1", "yes", "on", "TRUE"])
[email protected]("run_mode", ["oss", "saas"])
+def test_enabled_when_flag_and_supported_run_mode(monkeypatch, flag, run_mode):
+ monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)
+ monkeypatch.setenv("OPENRAG_INGEST_PREVIEW_ENABLED", flag)
+ assert is_ingest_preview_enabled() is True
+
+
+def test_flag_does_not_enable_on_prem(monkeypatch):
+ """The flag is AND-ed with the run mode: on_prem is never eligible."""
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "on_prem")
+ monkeypatch.setenv("OPENRAG_INGEST_PREVIEW_ENABLED", "true")
+ assert is_ingest_preview_enabled() is False