OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import asyncio
import mimetypes
import os
import time
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any, Literal
from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
from observability.spans import trace_root, trace_span
from services.search_cache import invalidate_corpus_caches
from session_manager import AnonymousUser
from utils.document_processing import (
extract_relevant,
process_text_file,
resplit_chunks_character_windows,
split_chunks_by_max_tokens,
)
from utils.file_utils import (
auto_cleanup_tempfile,
clean_connector_filename,
get_file_extension,
get_filename_aliases,
langflow_safe_filename_and_mimetype,
)
from utils.filename_claims import claim_holder, filename_claims
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
from utils.opensearch_queries import build_owned_filename_query, build_replace_filename_query
from .tasks import FileTask, TaskStatus, UploadTask
logger = get_logger(__name__)
DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"
TEXT_PARSER_LABEL = "Text Parser"
DUPLICATE_FILENAME_WARNING = "A file with this name already exists."
def _docling_counts(doc: Any) -> dict[str, int]:
"""What Docling produced, for the parse span: element counts, never text."""
if not isinstance(doc, dict):
return {}
return {key: len(doc.get(key) or ()) for key in ("pages", "texts", "tables", "pictures")}
def _chunk_type_counts(chunks: list[dict]) -> dict[str, int]:
"""Chunk count, and per type (text/table/picture) where non-zero."""
counts: dict[str, int] = {"chunks": len(chunks)}
for chunk in chunks:
key = f"chunks_{chunk.get('type') or 'unknown'}"
counts[key] = counts.get(key, 0) + 1
return counts
def _prompt_tokens(response: Any) -> int:
"""Input tokens an embedding response reports, or 0."""
usage = response.get("usage") if isinstance(response, dict) else None
if not isinstance(usage, dict):
return 0
tokens = usage.get("prompt_tokens") or usage.get("total_tokens") or 0
return tokens if isinstance(tokens, int) else 0
# Both outcomes finish the file the same way — SKIPPED, counted successful,
# duplicate warning — but only "skip" means a document with this name is
# actually indexed. "skip_in_flight" means another file in this batch is still
# writing it, which is a different fact about the index.
DUPLICATE_SKIP_ACTIONS = ("skip", "skip_in_flight")
DUPLICATE_CONTENT_WARNING = (
"Identical content already exists in the knowledge base under a different filename."
)
if TYPE_CHECKING:
from connectors.base import DocumentACL
def _verification_client(fallback_client):
"""Client for post-ingestion verification ("did the chunks land in the
index?"). That is a system integrity check, not a user-visibility check,
so prefer the platform writer client: it does not depend on the JWT/JWKS
trust chain that user-scoped clients need (OpenSearch loads the backend's
JWKS lazily, so the first user-JWT queries after a cold start can 401).
Falls back to the caller's client when the writer is unavailable."""
return clients.opensearch if clients.opensearch is not None else fallback_client
def resolve_shared_owner_fields(
user_id: str | None,
owner_name: str | None,
owner_email: str | None,
shared: bool,
) -> tuple[str | None, str | None, str | None]:
"""Return (owner, owner_name, owner_email) for indexing.
When shared=True, owner is None so the indexed chunk omits the owner field
entirely, triggering the OpenSearch DLS must_not-exists-owner clause that
makes the document visible to all users in the instance. owner_name and
owner_email are set to AnonymousUser values, matching how default/sample
documents are loaded.
"""
if shared:
_anon = AnonymousUser()
return None, _anon.name, _anon.email
return user_id, owner_name, owner_email
class TaskProcessor:
"""Base class for task processors with shared processing logic"""
def __init__(self, document_service=None, models_service=None, docling_service=None):
self.document_service = document_service
self.models_service = models_service
self.docling_service = docling_service
async def check_document_exists(
self,
file_hash: str,
opensearch_client,
on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",
*,
wait_for_visibility: bool = False,
field: str = "document_id",
) -> bool:
"""
Check if a document with the given hash already exists in OpenSearch.
Consolidated hash checking for all processors.
``on_error`` picks the answer when OpenSearch stays unreachable after
retries — the check is ambiguous then, and the safe default differs by
caller:
* ``"assume_missing"`` (dedupe callers): safer to reprocess than skip.
* ``"assume_exists"`` (post-ingestion verification callers): an
infrastructure error must not fail a file that Langflow already
reported as ingested.
When ``wait_for_visibility`` is True, an empty result is retried a few
times with backoff before concluding the document is absent. This is for
post-ingest verification: chunks that were just written may not be
searchable yet within OpenSearch's near-real-time refresh window
(default ~1s), and the user-scoped client cannot force an
``indices:admin/refresh`` (it lacks the privilege).
"""
max_retries = 3
retry_delay = 1.0
# Some deployments' indices predate connector_file_id's addition to the
# explicit mapping (config/settings.py), so OpenSearch dynamically
# mapped it as analyzed `text` (with a `.keyword` multi-field) instead
# of the intended `keyword` type. A plain term query against such a
# field tokenizes the query value and rarely matches the raw id, so
# also match its `.keyword` multi-field. document_id has always been
# explicitly `keyword` since index creation and never has this issue.
query: dict[str, Any]
if field == "connector_file_id":
query = {
"bool": {
"should": [
{"term": {field: file_hash}},
{"term": {f"{field}.keyword": file_hash}},
]
}
}
else:
query = {"term": {field: file_hash}}
for attempt in range(max_retries):
try:
response = await opensearch_client.search(
index=get_index_name(),
body={
"size": 1,
"_source": False,
"query": query,
},
)
hits = response.get("hits", {}).get("hits", [])
if hits:
return True
# No hits. For post-ingest verification, the document may not be
# visible yet within the near-real-time refresh window — retry.
if wait_for_visibility and attempt < max_retries - 1:
await asyncio.sleep(retry_delay)
retry_delay *= 2
continue
return False
except (TimeoutError, Exception) as e:
if attempt == max_retries - 1:
logger.error(
"OpenSearch exists check failed after retries",
file_hash=file_hash,
error=str(e),
attempt=attempt + 1,
)
if on_error == "assume_exists":
logger.warning(
"Exists check inconclusive due to connection issues; "
"assuming document exists",
file_hash=file_hash,
)
return True
# Safer to reprocess than skip for dedupe callers.
logger.warning(
"Assuming document doesn't exist due to connection issues",
file_hash=file_hash,
)
return False
else:
logger.warning(
"OpenSearch exists check failed, retrying",
file_hash=file_hash,
error=str(e),
attempt=attempt + 1,
retry_in=retry_delay,
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
return on_error == "assume_exists"
async def check_filename_exists(
self,
filename: str,
opensearch_client,
*,
wait_for_visibility: bool = False,
) -> bool:
"""
Check if a document with the given filename already exists in OpenSearch.
Returns True if any chunks with this filename exist.
When ``wait_for_visibility`` is True, an empty result is retried a few
times with backoff before concluding the document is absent. This is for
post-ingest verification: chunks that were just written may not be
searchable yet within OpenSearch's near-real-time refresh window
(default ~1s), and the user-scoped client cannot force an
``indices:admin/refresh`` (it lacks the privilege).
"""
from utils.opensearch_filenames import find_existing_filenames
max_retries = 3
retry_delay = 1.0
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
return False
for attempt in range(max_retries):
try:
# One bulk existence check covering every alias — the shared
# query semantic used by all duplicate-detection altitudes.
if await find_existing_filenames(
candidate_filenames, opensearch_client, get_index_name()
):
return True
# No alias exists. For post-ingest verification, the document
# may not be visible yet within the near-real-time refresh
# window — re-check after a short delay.
if wait_for_visibility and attempt < max_retries - 1:
await asyncio.sleep(retry_delay)
retry_delay *= 2
continue
return False
except (TimeoutError, Exception) as e:
if attempt == max_retries - 1:
logger.error(
"OpenSearch filename check failed after retries",
filename=filename,
error=str(e),
attempt=attempt + 1,
)
# On final failure, assume document doesn't exist (safer to reprocess than skip)
logger.warning(
"Assuming filename doesn't exist due to connection issues",
filename=filename,
)
return False
else:
logger.warning(
"OpenSearch filename check failed, retrying",
filename=filename,
error=str(e),
attempt=attempt + 1,
retry_in=retry_delay,
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
return False
async def resolve_duplicate_filename(
self,
filename: str,
opensearch_client,
*,
replace: bool,
owner_user_id: str | None,
allow_anonymous_delete: bool,
shared: bool = False,
claim_holder: str | None = None,
) -> Literal["proceed", "skip", "skip_in_flight", "replaced"]:
"""Single duplicate-filename policy shared by every processor.
Checks whether a document with this filename (or one of its aliases)
is already indexed and applies the caller's replace decision:
* ``"proceed"`` — no duplicate; continue ingestion.
* ``"skip"`` — a document with this name is indexed and
``replace`` is False; the caller should finish via
``mark_duplicate_skipped``.
* ``"skip_in_flight"`` — another file in this batch holds the name and
has not finished writing it. Finishes the same way,
but nothing is indexed under the name *yet*, so a
caller must not treat it as a statement about what
is in the index (see ``_reconcile_shared_owner``).
* ``"replaced"`` — duplicate and ``replace`` is True; the existing
chunks were deleted and the index refreshed, so
ingestion can continue.
``claim_holder`` identifies this file's run (see ``utils.filename_claims``)
and holds the name for its duration, so a second file heading for the
same name inside the same batch resolves as a duplicate instead of
racing the index check. TaskService releases the claim when the file
reaches a terminal state. Callers that pass None get the OpenSearch
policy alone — every processor passes one.
``allow_anonymous_delete`` is handed to ``delete_document_by_filename``
and, like there, has no default: see that docstring.
"""
if claim_holder is not None and not filename_claims.claim(
claim_holder, filename, owner_user_id=owner_user_id, shared=shared
):
# Another file in flight is already heading for this name; whichever
# of them lands first is the one this name belongs to.
return "skip_in_flight"
if not await self.check_filename_exists(filename, opensearch_client):
return "proceed"
if not replace:
return "skip"
logger.info(f"Replacing existing document: {filename}")
deleted = await self.delete_document_by_filename(
filename,
opensearch_client,
owner_user_id=owner_user_id,
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
if deleted == 0:
logger.warning(
"Replacement requested but deletion removed no chunks",
filename=filename,
)
return "skip"
# Refresh so the delete is visible before re-ingest. refresh is
# index-wide (indices:admin/refresh) and cannot be DLS-scoped, so it
# must run under the admin/service client, not the user client.
try:
await clients.opensearch.indices.refresh(index=get_index_name())
except Exception as refresh_error:
logger.warning(
"Failed to refresh index after delete",
error=str(refresh_error),
)
return "replaced"
@staticmethod
def _claim_holder(upload_task: UploadTask, file_task: FileTask) -> str:
"""Identity of this file's run, for the in-flight filename claim."""
return claim_holder(upload_task.task_id, file_task.file_path)
def mark_duplicate_skipped(self, upload_task: UploadTask, file_task: FileTask) -> None:
"""Uniform terminal state for a duplicate that was not replaced:
SKIPPED, counted toward successful files, with a warning the task view
surfaces. A declined replacement is a chosen outcome, not an error."""
file_task.status = TaskStatus.SKIPPED
file_task.error = None
file_task.result = {
"status": "skipped",
"reason": "duplicate_filename",
"warning": DUPLICATE_FILENAME_WARNING,
}
file_task.updated_at = time.time()
upload_task.successful_files += 1
async def delete_document_by_filename(
self,
filename: str,
opensearch_client,
owner_user_id: str | None = None,
shared: bool = False,
*,
allow_anonymous_delete: bool,
) -> int:
"""Delete all chunks of a document with the given filename from
OpenSearch. Returns the number of chunks deleted.
``shared`` describes how the replacement is about to be *written*, not
what is already indexed, so it must not narrow what we delete: with an
owner in hand, and the permission below, the scope is "owned by this
user OR ownerless".
Choosing an owner-only scope for a shared document matched none of its
chunks, and the caller read that zero as "nothing to replace" and
skipped the file — leaving the stale copy in the index even though the
duplicate check (which is owner-agnostic) had just found it and the user
had confirmed the overwrite.
``allow_anonymous_delete`` is the caller's resolved
``knowledge:delete:anonymous``. Ownerless chunks are visible to everyone
in the instance, so replacing a document that turns out to be one is a
deletion of shared content: without that permission the scope stays
owner-only and someone else's shared document is left alone (the file
then resolves as a duplicate the user may not replace). It has no
default on purpose. It used to default to True, and the entry points
that never resolved the permission — local upload, S3, the Langflow
path — inherited that silently, so any user could replace a shared
document through them. Every caller now has to say which it is, down
to the processor constructors and the task-service helpers that feed
them; one acting for the system rather than for a user passes True and
says why.
``shared`` does not stand in for it. An explicit shared sync already
holds the permission (connector_sync refuses one without it), but on
the re-sync paths ``shared`` is inferred from what is indexed, and an
inferred answer authorizes nothing: honouring it let a caller without
the permission replace a shared document simply by re-syncing it."""
from config.settings import clients, get_index_name
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
from utils.opensearch_queries import (
build_anonymous_filename_query,
build_owned_filename_query,
build_replace_filename_query,
)
try:
write_client = clients.opensearch
if write_client is None:
raise RuntimeError("Backend OpenSearch write client is unavailable")
if not owner_user_id:
if shared:
def build_query(fname, _owner):
return build_anonymous_filename_query(fname)
else:
logger.warning(
"Skipped delete_by_filename because owner_user_id is missing",
filename=filename,
)
return 0
elif allow_anonymous_delete:
build_query = build_replace_filename_query
else:
build_query = build_owned_filename_query
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
logger.info(
"Skipped delete_by_filename because filename input is empty",
filename=filename,
)
return 0
deleted_count = 0
for candidate in candidate_filenames:
document_ids = await collect_visible_document_ids(
opensearch_client,
index=get_index_name(),
query=build_query(candidate, owner_user_id),
)
deleted_count += await delete_document_ids(
write_client,
index=get_index_name(),
document_ids=document_ids,
)
logger.info(
"Deleted existing document chunks", filename=filename, deleted_count=deleted_count
)
return deleted_count
except Exception as e:
logger.error("Failed to delete existing document", filename=filename, error=str(e))
raise
async def _delete_connector_chunks(
self,
file_id: str,
opensearch_client,
owner_user_id: str,
keep_filenames: list[str] | None = None,
shared: bool = False,
connector_type: str | None = None,
*,
ignore_errors: bool = True,
) -> int:
"""Delete indexed chunks for a connector file by its STABLE id.
Deletion semantics (dual-field id match, connector/owner/shared scoping,
rename ``keep_filenames``) live in ``connectors.chunk_cleanup``.
Rename cleanup is best-effort (``ignore_errors=True``): logs and returns
0 so a miss never fails an ingest that can still re-index the new name.
Source-deleted cleanup passes ``ignore_errors=False`` so a real index
failure fails the file task instead of reporting success.
``connector_type`` scopes the match to one connector type — the same
value the chunks were indexed under — so an id that collides with a
different connector's id can't take its chunks down with it.
Callers pass the sharing mode resolved from what is actually indexed
(``_resolve_shared``), so this deletes under exactly that layout —
ownerless chunks for a share-all file, owner-scoped ones otherwise.
It must NOT widen to "owned OR ownerless" on top of that: the resolved
mode already matches the file, and widening would let a sync of an owned
file reach ownerless chunks that another user's connection ingested,
with none of the ``knowledge:delete:anonymous`` gating that deliberate
shared deletion goes through.
"""
from connectors.chunk_cleanup import delete_connector_file_chunks
if not file_id:
return 0
try:
return await delete_connector_file_chunks(
[file_id],
opensearch_client,
connector_type=connector_type,
owner_user_id=owner_user_id,
shared=shared,
keep_filenames=keep_filenames,
)
except Exception as e:
logger.error(
"Failed to delete connector chunks",
file_id=file_id,
error=str(e),
)
if ignore_errors:
return 0
raise
async def process_document_standard(
self,
file_path: str,
file_hash: str,
owner_user_id: str = None,
original_filename: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
file_size: int = None,
connector_type: str = "local",
embedding_model: str = None,
chunk_size: int = None,
chunk_overlap: int = None,
is_sample_data: bool = False,
acl: "DocumentACL | None" = None,
connector_file_id: str | None = None,
ocr: bool | None = None,
picture_descriptions: bool | None = None,
shared: bool = False,
document_id: str | None = None,
replace_existing: bool = False,
source_url: str | None = None,
web_source_id: str | None = None,
web_page_id: str | None = None,
web_page_depth: int | None = None,
root_source_url: str | None = None,
canonical_url: str | None = None,
before_index_write: Callable[[], Awaitable[object]] | None = None,
):
"""
Standard processing pipeline for non-Langflow processors:
docling conversion + embeddings + OpenSearch indexing.
Args:
embedding_model: Embedding model to use (defaults to the current
embedding model from settings)
chunk_size: Optional character window size for re-splitting extracted
chunks (non-Langflow path, e.g. connector UI ``chunkSize``).
chunk_overlap: Overlap between windows; must be less than ``chunk_size``.
acl: DocumentACL instance with access control information
ocr: Per-request OCR override (None = use global config).
picture_descriptions: Per-request picture descriptions override.
"""
from services.document_service import chunk_texts_for_embeddings
# Use provided embedding model or configured model.
# get_embedding_model() returns empty string when Langflow ingest is enabled,
# but OpenRAG processors still need a concrete embedding model.
config = get_openrag_config()
configured_embedding_model = config.knowledge.embedding_model
embedding_provider = getattr(config.knowledge, "embedding_provider", None) or "openai"
embedding_model = embedding_model or configured_embedding_model or get_embedding_model()
if chunk_size is None:
chunk_size = config.knowledge.chunk_size
if chunk_overlap is None:
chunk_overlap = config.knowledge.chunk_overlap
stable_document_id = document_id or file_hash
file_ext = os.path.splitext(file_path)[1].lower()
# One trace per file, with a span per stage; no-ops while tracing is off.
with trace_root(
"ingest.file",
user_id=owner_user_id,
tags=["ingest_path:openrag", f"connector:{connector_type}"],
attributes={
"document_id": stable_document_id,
"ingest_path": "openrag",
"connector_type": connector_type,
"embedding_model": embedding_model,
"embedding_provider": embedding_provider,
"file.extension": file_ext or None,
"file.size": file_size,
"sample_data": is_sample_data or None,
},
) as ingest_trace:
ingest_trace.set_content(filename=original_filename or os.path.basename(file_path))
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
owner_user_id, jwt_token
)
# URL sources own normalized-content deduplication. A page re-sync
# deliberately replaces chunks under its stable document ID.
if not replace_existing and await self.check_document_exists(
stable_document_id, opensearch_client
):
ingest_trace.set(status="unchanged")
return {"status": "unchanged", "id": stable_document_id}
logger.info(
"Processing document with embedding model",
embedding_model=embedding_model,
file_hash=stable_document_id,
)
# Check if this is a .txt or .md file - use simple processing instead of docling
if file_ext in (".txt", ".md"):
# Simple text file processing without docling
logger.info(
"Processing as plain text file (bypassing docling)",
file_path=file_path,
file_hash=file_hash,
)
with trace_span("ingest.parse", parser=TEXT_PARSER_LABEL) as parse:
slim_doc = process_text_file(file_path)
parse.set(chunks=len(slim_doc["chunks"]))
slim_doc["parser"] = TEXT_PARSER_LABEL
else:
with trace_span(
"ingest.parse",
parser=DOCLING_PARSER_LABEL,
# Unset per-request options fall back to config, as in Docling.
ocr=ocr if ocr is not None else getattr(config.knowledge, "ocr", None),
picture_descriptions=picture_descriptions
if picture_descriptions is not None
else getattr(config.knowledge, "picture_descriptions", None),
table_structure=getattr(config.knowledge, "table_structure", None),
) as parse:
full_doc = await self.docling_service.convert_file(
file_path,
user_id=owner_user_id,
auth_header=jwt_token,
ocr=ocr,
picture_descriptions=picture_descriptions,
)
parse.set(**_docling_counts(full_doc))
with trace_span("ingest.extract") as extract:
slim_doc = extract_relevant(full_doc)
extract.set(**_chunk_type_counts(slim_doc["chunks"]))
slim_doc["parser"] = DOCLING_PARSER_LABEL
ingest_trace.set(mimetype=slim_doc.get("mimetype"))
# Override filename with original_filename if provided
if original_filename:
slim_doc["filename"] = original_filename
if chunk_size is not None:
try:
cs = int(chunk_size)
except (TypeError, ValueError):
cs = 0
if cs > 0:
try:
co = int(chunk_overlap) if chunk_overlap is not None else 0
except (TypeError, ValueError):
co = 0
if co < cs:
with trace_span(
"ingest.resplit",
chunk_size=cs,
chunk_overlap=max(0, co),
chunks_in=len(slim_doc["chunks"]),
) as resplit:
slim_doc["chunks"] = resplit_chunks_character_windows(
slim_doc["chunks"], cs, max(0, co)
)
resplit.set(chunks_out=len(slim_doc["chunks"]))
litellm_embedding_model = (
await self.models_service.get_litellm_model_name(
embedding_model,
provider=embedding_provider,
)
if self.models_service is not None
else embedding_model
)
litellm_model_lower = litellm_embedding_model.lower() if litellm_embedding_model else ""
if "watsonx" in litellm_model_lower:
max_tokens = 500
elif "ollama" in litellm_model_lower:
max_tokens = 2000
else:
max_tokens = 8000
with trace_span(
"ingest.token_cap", max_tokens=max_tokens, chunks_in=len(slim_doc["chunks"])
) as token_cap:
# Filter out chunks with empty or whitespace-only text before generating embeddings.
# This ensures the length of chunks matches the length of the embeddings array,
# since chunk_texts_for_embeddings also drops empty texts.
slim_doc["chunks"] = [
c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
]
non_empty = len(slim_doc["chunks"])
# Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
slim_doc["chunks"] = split_chunks_by_max_tokens(
slim_doc["chunks"], max_tokens, litellm_embedding_model
)
# Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting
slim_doc["chunks"] = [
c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
]
token_cap.set(non_empty=non_empty, chunks_out=len(slim_doc["chunks"]))
texts = [c["text"] for c in slim_doc["chunks"]]
text_batches = chunk_texts_for_embeddings(
texts, max_tokens=max_tokens, model=litellm_embedding_model
)
embeddings = []
# Every provider goes through the gateway, not just the aliased ones.
# The gateway resolves credentials per call from config; the direct
# client resolved them from process-global environment written once per
# process, which is never unset when a provider is deconfigured and is
# shared by every provider in the process. Gating on "is this provider
# aliased?" also meant the correctness of ingestion depended on an
# unrelated routing detail.
from services.llm_gateway import embeddings as gateway_embeddings
from services.llm_gateway import qualified_model_id
gateway_model = qualified_model_id(embedding_provider, embedding_model)
with trace_span("ingest.embed", batches=len(text_batches), inputs=len(texts)) as embed:
prompt_tokens = 0
for batch in text_batches:
response = await gateway_embeddings({"model": gateway_model, "input": batch})
data = response.get("data", [])
embeddings.extend(
[
item["embedding"] if isinstance(item, dict) else item.embedding
for item in data
]
)
prompt_tokens += _prompt_tokens(response)
embed.set(
vectors=len(embeddings),
dimensions=len(embeddings[0]) if embeddings else None,
prompt_tokens=prompt_tokens or None,
)
if not embeddings or len(embeddings) == 0:
logger.error(
"No embeddings generated — document may be empty or unreadable",
file_hash=file_hash,
embedding_model=embedding_model,
)
ingest_trace.fail("NoEmbeddings")
return {
"status": "error",
"error": "No text content could be extracted from document",
}
from services.document_index_writer import (
DocumentIndexChunk,
DocumentIndexContext,
DocumentIndexWriter,
)
document_index_writer = getattr(self.document_service, "document_index_writer", None)
if document_index_writer is None:
document_index_writer = DocumentIndexWriter()
# Owner is always the authenticated uploading/syncing user unless shared=True,
# in which case owner fields are omitted so DLS makes the doc visible to all users.
owner, owner_name, owner_email = resolve_shared_owner_fields(
owner_user_id, owner_name, owner_email, shared
)
if acl:
allowed_users = acl.allowed_users or []
allowed_groups = acl.allowed_groups or []
allowed_principals = acl.allowed_principals or []
allowed_principal_labels = acl.allowed_principal_labels or []
else:
allowed_users = []
allowed_groups = []
allowed_principals = []
allowed_principal_labels = []
filename = original_filename if original_filename else slim_doc["filename"]
index_context = DocumentIndexContext(
document_id=stable_document_id,
filename=filename,
mimetype=slim_doc["mimetype"],
embedding_model=embedding_model,
embedding_provider=embedding_provider,
owner=owner,
owner_name=owner_name,
owner_email=owner_email,
file_size=file_size,
connector_type=connector_type,
source_url=source_url,
record_kind="web_page" if web_source_id else None,
web_source_id=web_source_id,
web_page_id=web_page_id,
web_page_depth=web_page_depth,
root_source_url=root_source_url,
canonical_url=canonical_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,
allowed_principal_labels=allowed_principal_labels,
is_sample_data=is_sample_data,
)
parser_name = slim_doc.get("parser")
if not parser_name:
if file_ext in (".txt", ".md"):
parser_name = TEXT_PARSER_LABEL
else:
parser_name = DOCLING_PARSER_LABEL
chunk_metadata = {"parser": parser_name}
if chunk_size is not None:
chunk_metadata["chunk_size"] = chunk_size
if chunk_overlap is not None:
chunk_metadata["chunk_overlap"] = chunk_overlap
if connector_file_id:
chunk_metadata["connector_file_id"] = connector_file_id
index_chunks = [
DocumentIndexChunk(
chunk_id=f"{stable_document_id}_{i}",
text=chunk["text"],
vector=vect,
page=chunk["page"],
metadata=chunk_metadata,
)
for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True))
]
with trace_span("ingest.index") as index:
# This guard is intentionally outside the stale-cleanup handler.
# A URL source can be deleted while its crawl is still processing;
# in that case indexing must stop rather than recreating chunks for
# the deleted source.
if before_index_write is not None:
await before_index_write()
# Clear stale chunks from a prior indexing of this document. Chunks are
# stored under ids {file_hash}_{i}; if the new chunk count is lower
# than the prior one, trailing chunks would otherwise survive the
# writer's idempotent upsert.
# DLS-safe: enumerate visible chunk ids with the scoped user client,
# then delete concrete ids with the trusted backend client.
try:
from utils.opensearch_delete import (
collect_visible_document_ids,
delete_document_ids,
)
write_client = clients.opensearch
if write_client is None:
raise RuntimeError("Backend OpenSearch write client is unavailable")
stale_chunk_ids = await collect_visible_document_ids(
opensearch_client,
index=get_index_name(),
query={"term": {"document_id": stable_document_id}},
)
await delete_document_ids(
write_client,
index=get_index_name(),
document_ids=stale_chunk_ids,
refresh=True,
)
index.set(stale_chunks_deleted=len(stale_chunk_ids))
except Exception as e:
logger.warning(
"Failed to clear stale chunks before re-index; proceeding",
file_hash=stable_document_id,
error=str(e),
)
index.set(stale_cleanup_failed=True)
await document_index_writer.index_chunks(index_context, index_chunks, final=True)
index.set(chunks_indexed=len(index_chunks))
ingest_trace.set(status="indexed", chunks=len(index_chunks))
return {
"status": "indexed",
"id": stable_document_id,
"chunk_count": len(index_chunks),
}
async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:
"""
Process a single item in the task.
This is a base implementation that should be overridden by subclasses.
When TaskProcessor is used directly (not via subclass), this method
is not called - only the utility methods like process_document_standard
are used.
Args:
upload_task: The overall upload task
item: The item to process (could be file path, file info, etc.)
file_task: The specific file task to update
"""
raise NotImplementedError(
"process_item should be overridden by subclasses when used in task processing"
)
class DocumentFileProcessor(TaskProcessor):
"""Default processor for regular file uploads"""
def __init__(
self,
document_service,
models_service,
owner_user_id: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
is_sample_data: bool = False,
connector_type: str = "local",
docling_service=None,
replace_duplicates: bool = False,
session_manager=None,
settings: dict | None = None,
*,
allow_anonymous_delete: bool,
):
super().__init__(
document_service,
models_service,
docling_service=docling_service
or (document_service.docling_service if document_service else None),
)
self.owner_user_id = owner_user_id
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
self.is_sample_data = is_sample_data
self.connector_type = connector_type
self.replace_duplicates = replace_duplicates
self.session_manager = session_manager or (
document_service.session_manager if document_service else None
)
self.settings = settings
# The uploading user's resolved knowledge:delete:anonymous; see
# delete_document_by_filename.
self.allow_anonymous_delete = allow_anonymous_delete
if self.session_manager is None:
raise ValueError("session_manager is required for DocumentFileProcessor")
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a regular file path using consolidated methods"""
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
try:
# Use the ORIGINAL filename stored in file_task (not the transformed temp path)
# This ensures we check/store the original filename with spaces, etc.
original_filename = file_task.filename or os.path.basename(item)
# Check if document with same filename already exists
if self.session_manager is None:
raise ValueError("session_manager is required to get OpenSearch client")
opensearch_client = self.session_manager.get_user_opensearch_client(
self.owner_user_id, self.jwt_token
)
duplicate_action = await self.resolve_duplicate_filename(
original_filename,
opensearch_client,
replace=self.replace_duplicates,
owner_user_id=self.owner_user_id,
claim_holder=self._claim_holder(upload_task, file_task),
allow_anonymous_delete=self.allow_anonymous_delete,
)
if duplicate_action in DUPLICATE_SKIP_ACTIONS:
self.mark_duplicate_skipped(upload_task, file_task)
return
# Compute hash
file_hash = hash_id(item)
# Chunks are indexed with document_id=file_hash (see
# process_document_standard -> DocumentIndexContext), so record it on
# the file_task for preview-mode index proof lookups.
file_task.document_id = file_hash
# Get file size
try:
file_size = os.path.getsize(item)
except Exception:
file_size = 0
# Parse ACL from settings if present
from connectors.base import DocumentACL
acl = None
if self.settings and (
self.settings.get("allowed_users") is not None
or self.settings.get("allowed_groups") is not None
):
acl = DocumentACL(
owner=self.owner_user_id,
allowed_users=self.settings.get("allowed_users", []),
allowed_groups=self.settings.get("allowed_groups", []),
)
standard_kwargs: dict[str, Any] = {}
if self.settings:
s = self.settings
em = s.get("embeddingModel")
if isinstance(em, str) and em.strip():
standard_kwargs["embedding_model"] = em.strip()
for ui_key, param in (
("chunkSize", "chunk_size"),
("chunkOverlap", "chunk_overlap"),
):
raw = s.get(ui_key)
if raw is not None:
try:
standard_kwargs[param] = int(raw)
except (TypeError, ValueError):
pass
config = get_openrag_config()
standard_kwargs["ocr"] = config.knowledge.ocr
standard_kwargs["picture_descriptions"] = config.knowledge.picture_descriptions
# Use consolidated standard processing
result = await self.process_document_standard(
file_path=item,
file_hash=file_hash,
owner_user_id=self.owner_user_id,
original_filename=original_filename,
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=file_size,
connector_type=self.connector_type,
is_sample_data=self.is_sample_data,
acl=acl,
**standard_kwargs,
)
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
file_task.error = result.get("error") or "Failed to process document"
file_task.updated_at = time.time()
upload_task.failed_files += 1
else:
file_task.status = TaskStatus.COMPLETED
file_task.result = result
file_task.updated_at = time.time()
upload_task.successful_files += 1
except Exception as e:
file_task.status = TaskStatus.FAILED
file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
class ConnectorFileProcessor(TaskProcessor):
"""Processor for connector file uploads"""
def __init__(
self,
connector_service,
connection_id: str,
files_to_process: list,
user_id: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
document_service=None,
models_service=None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
connector_type: str | None = None,
preview_mode: bool = False,
shared: bool | None = False,
*,
allow_anonymous_delete: bool,
):
super().__init__(
document_service=document_service,
models_service=models_service,
docling_service=document_service.docling_service if document_service else None,
)
self.connector_service = connector_service
self.connection_id = connection_id
self.files_to_process = files_to_process
self.user_id = user_id
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
self.ingest_settings = ingest_settings
self.replace_duplicates = replace_duplicates
self.connector_type = connector_type
self.preview_mode = preview_mode
self.shared = shared
# The syncing user's resolved knowledge:delete:anonymous, threaded down
# to the replace path so an overwrite cannot delete a shared document
# the user is not allowed to delete.
self.allow_anonymous_delete = allow_anonymous_delete
async def _indexed_shared_state(
self,
file_id: str,
opensearch_client,
connector_type: str | None,
) -> bool | None:
"""Whether this connector file is ALREADY indexed as shared (ownerless).
Returns True when the indexed chunks carry no ``owner`` (the share-all
layout), False when they carry one, and None when the file isn't indexed
yet or the lookup failed — callers fall back to their own default.
Read with the caller's (user-scoped) client, so DLS keeps documents we
cannot see out of the answer.
"""
if not file_id:
return None
from connectors.chunk_cleanup import build_connector_file_chunks_query
try:
response = await opensearch_client.search(
index=get_index_name(),
body={
"size": 1,
"_source": ["owner"],
"query": build_connector_file_chunks_query(
[file_id], connector_type=connector_type
),
},
)
except Exception as e:
logger.warning(
"Could not read indexed sharing state for connector file",
file_id=file_id,
error=str(e),
)
return None
hits = response.get("hits", {}).get("hits", [])
if not hits:
return None
return not hits[0].get("_source", {}).get("owner")
async def _resolve_shared(
self,
file_id: str,
opensearch_client,
connector_type: str | None,
) -> bool:
"""Resolve the sharing mode to index this file under.
``self.shared`` is the caller's explicit intent — the connector upload
UI's "Make documents available to all users" toggle. It is None on the
re-sync paths, which run with no user in the loop and no record of what
each file was ingested as; those inherit whatever is already in the
index. Without that inheritance a sync would treat a shared file as
private, and every owner-scoped step downstream (the replace-delete, the
chunk cleanup, the owner fields on re-index) would either match nothing
or silently un-share the document.
Files not yet indexed fall back to private, matching the previous
default for a caller that expressed no intent.
"""
if self.shared is not None:
return self.shared
return bool(await self._indexed_shared_state(file_id, opensearch_client, connector_type))
async def _reconcile_shared_owner(self, filename: str, shared: bool) -> None:
"""Update owner fields on already-indexed chunks for `filename` to match
the `shared` setting resolved for this file.
Called where the document indexed under this name is THIS file's own —
the unchanged-content path, and a skip against a real duplicate. Never
for a "skip_in_flight": there the name belongs to another file that is
still writing it, and this query matches by filename, so reconciling
would stamp this file's sharing answer onto that one's chunks.
On those paths the file's content and name haven't changed since a prior
sync, but the connector's "Make documents available to all users"
setting may have been toggled since. Without this, those chunks would
keep whatever owner they
got on their original ingest forever, since a byte-identical re-sync
never reaches resolve_shared_owner_fields(). Scoped to chunks owned by
this user — and to ownerless ones only under the same
``knowledge:delete:anonymous`` boundary delete_document_by_filename
uses — so it can't touch another user's document that happens to share
this filename.
That boundary matters here even more than it does for a delete. An
ownerless document is visible to the whole instance, and this script
writes an owner onto what it matches: without the permission check, a
private sync of a file whose name collides with a shared document would
quietly claim that document for the syncing user, taking it out of
everyone else's view. ``_indexed_shared_state`` cannot prevent it — it
keys on the connector file id, and a collision by definition comes from
a document this connector never ingested, so the lookup returns None and
``shared`` falls back to the sync's own (private) intent.
`shared` comes from _resolve_shared, never straight from self.shared: on
a re-sync it reflects the file's current indexed state, so this is a
no-op rather than an unrequested ownership change.
"""
write_client = clients.opensearch
if write_client is None:
return
owner, owner_name, owner_email = resolve_shared_owner_fields(
self.user_id, self.owner_name, self.owner_email, shared
)
# A shared write already cleared the permission upstream (connector_sync
# rejects shared syncs without it), so it keeps the wider scope.
build_query = (
build_replace_filename_query
if shared or self.allow_anonymous_delete
else build_owned_filename_query
)
for candidate in get_filename_aliases(filename):
try:
await write_client.update_by_query(
index=get_index_name(),
body={
"query": build_query(candidate, self.user_id),
"script": {
"source": """
if (params.shared) {
ctx._source.remove('owner');
} else {
ctx._source.owner = params.owner;
}
ctx._source.owner_name = params.owner_name;
ctx._source.owner_email = params.owner_email;
""",
"params": {
"shared": shared,
"owner": owner,
"owner_name": owner_name,
"owner_email": owner_email,
},
},
},
refresh=True,
)
invalidate_corpus_caches()
except Exception as e:
logger.warning(
"Failed to reconcile owner fields for skipped duplicate",
filename=candidate,
error=str(e),
)
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using unified methods"""
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
try:
file_id = item # item is the connector file ID
# Get the connector and connection info
connector = await self.connector_service.get_connector(self.connection_id)
connection = await self.connector_service.connection_manager.get_connection(
self.connection_id
)
if not connector or not connection:
raise ValueError(f"Connection '{self.connection_id}' not found")
connector_type = self.connector_type or connection.connector_type
# Validate file extension early if filename is available
VALID_EXTENSIONS = {
"adoc",
"asciidoc",
"asc",
"bmp",
"csv",
"dotx",
"dotm",
"docm",
"docx",
"htm",
"html",
"jpeg",
"jpg",
"md",
"pdf",
"png",
"potx",
"ppsx",
"pptm",
"potm",
"ppsm",
"pptx",
"tiff",
"txt",
"xls",
"xlsx",
"xhtml",
"webp",
}
# Only pre-validate when we have a real filename. When the filename
# falls back to the connector file_id (e.g. a deletion event re-added
# by sync_specific_files, where no name is known), skip this check so
# the deletion reaches the 404 -> chunk-cleanup path below. Files that
# still exist are re-validated after download (see below).
if file_task.filename and file_task.filename != file_id:
ext = file_task.filename.split(".")[-1].lower() if "." in file_task.filename else ""
if ext not in VALID_EXTENSIONS:
file_task.status = TaskStatus.FAILED
file_task.error = f"The file '{file_task.filename}' has an incompatible type."
file_task.updated_at = time.time()
upload_task.failed_files += 1
return
# Get file content from connector
try:
document = await connector.get_file_content(file_id)
except (FileNotFoundError, ValueError) as e:
msg = str(e).lower()
if "not found" in msg or "404" in msg:
# File gone at source — remove its indexed chunks by the
# stable connector id (matches both connector_file_id and
# document_id) so it stops appearing in search/chat.
opensearch_client = (
self.document_service.session_manager.get_user_opensearch_client(
self.user_id, self.jwt_token
)
)
try:
deleted_chunks = await self._delete_connector_chunks(
file_id,
opensearch_client,
self.user_id,
shared=await self._resolve_shared(
file_id, opensearch_client, connector_type
),
connector_type=connector_type,
ignore_errors=False,
)
except Exception:
file_task.status = TaskStatus.FAILED
file_task.error = (
"File no longer exists at source, but removing it "
"from the index failed."
)
file_task.updated_at = time.time()
upload_task.failed_files += 1
return
logger.info(
"File no longer exists at source — removed from index",
file_id=file_id,
connection_id=self.connection_id,
deleted_chunks=deleted_chunks,
source_error=str(e),
)
# Successful cleanup: the file is gone at the source and its
# chunks were removed. Mark completed (not skipped) so the
# tasks view does not treat this as a warning. Enhanced
# listing still surfaces this reason so the dialog can show
# it as Removed instead of omitting it like a normal ingest.
file_task.status = TaskStatus.COMPLETED
file_task.error = None
file_task.result = {
"status": "completed",
"reason": "deleted_at_source",
"deleted_chunks": deleted_chunks,
"message": (
f"File no longer exists at source; removed from index "
f"({deleted_chunks} chunk(s) deleted)."
),
}
file_task.updated_at = time.time()
upload_task.successful_files += 1
return
raise
# Update filename in task once we have it from the connector
file_task.filename = clean_connector_filename(document.filename, document.mimetype)
# Re-check filename validation
name = file_task.filename or document.filename or ""
ext = name.split(".")[-1].lower() if "." in name else ""
if ext not in VALID_EXTENSIONS:
file_task.status = TaskStatus.FAILED
file_task.error = f"The file '{name}' has an incompatible type."
file_task.updated_at = time.time()
upload_task.failed_files += 1
return
if not self.user_id:
raise ValueError("user_id not provided to ConnectorFileProcessor")
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
self.user_id, self.jwt_token
)
# Resolve once, up front: every owner-scoped step below (the
# replace-delete, the rename cleanup, the owner fields written at
# index time) has to agree on how this file is shared, or the sync
# skips it and leaves the stale copy in place.
shared = await self._resolve_shared(document.id, opensearch_client, connector_type)
duplicate_action = await self.resolve_duplicate_filename(
file_task.filename,
opensearch_client,
replace=self.replace_duplicates,
owner_user_id=self.user_id,
shared=shared,
claim_holder=self._claim_holder(upload_task, file_task),
allow_anonymous_delete=self.allow_anonymous_delete,
)
if duplicate_action in DUPLICATE_SKIP_ACTIONS:
# Only for a real duplicate. The reconcile rewrites owner fields
# on every chunk with this filename, and "skip_in_flight" means
# the chunks under that name are being written right now by a
# DIFFERENT document, whose sharing state was resolved from its
# own file id and can differ from this one's. Reconciling then
# would stamp this file's answer onto another file's chunks.
if duplicate_action == "skip":
await self._reconcile_shared_owner(file_task.filename, shared)
self.mark_duplicate_skipped(upload_task, file_task)
return
# Rename cleanup: a connector file keeps a stable id across renames,
# but chunks are keyed by filename/content-hash, so a renamed file
# leaves its OLD-name chunks orphaned. Drop chunks for this id whose
# filename differs from the current one. If any were removed (a real
# rename), force a re-ingest below so the file is re-indexed under
# the new name instead of short-circuiting as "unchanged".
# Match against file_task.filename — the cleaned name the file is
# actually indexed under — so duplicate/rename detection lines up
# with how chunks are keyed.
renamed = (
await self._delete_connector_chunks(
document.id,
opensearch_client,
self.user_id,
keep_filenames=get_filename_aliases(file_task.filename),
shared=shared,
connector_type=connector_type,
)
> 0
)
# Create temporary file from document content
suffix = os.path.splitext(file_task.filename)[1]
if not suffix:
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
with open(tmp_path, "wb") as f:
f.write(document.content)
# Compute hash
file_hash = hash_id(tmp_path)
if not renamed and await self.check_document_exists(file_hash, opensearch_client):
await self._reconcile_shared_owner(file_task.filename, shared)
file_task.status = TaskStatus.COMPLETED
file_task.result = {"status": "unchanged", "id": file_hash}
file_task.updated_at = time.time()
upload_task.successful_files += 1
return
from config.settings import DISABLE_INGEST_WITH_LANGFLOW
if (
not DISABLE_INGEST_WITH_LANGFLOW
and self.connector_service.langflow_service is not None
):
# Delete existing chunks for this document before Langflow re-ingestion
try:
from utils.opensearch_delete import (
collect_visible_document_ids,
delete_document_ids,
)
# Match both fields: bucket-connector chunks carry the
# raw connector id in connector_file_id (document_id is
# a hash), while pre-migration chunks only have it in
# document_id.
chunk_ids = await collect_visible_document_ids(
opensearch_client,
index=get_index_name(),
query={
"bool": {
"should": [
{"term": {"document_id": document.id}},
{"term": {"connector_file_id": document.id}},
# See check_document_exists: some indices
# predate the explicit keyword mapping for
# this field.
{"term": {"connector_file_id.keyword": document.id}},
]
}
},
)
deleted_count = await delete_document_ids(
opensearch_client,
index=get_index_name(),
document_ids=chunk_ids,
refresh=True,
)
logger.info(
"Deleted existing chunks before Langflow re-ingestion",
document_id=document.id,
deleted_count=deleted_count,
)
except Exception as delete_err:
logger.warning(
"Failed to delete existing chunks before Langflow re-ingestion",
document_id=document.id,
error=str(delete_err),
)
# Ingest via unified Langflow pipeline (two-phase Docling + Langflow run)
langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(
file_task.filename, document.mimetype
)
file_tuple = (langflow_filename, document.content, processed_mimetype)
# Extract ACL information
allowed_users: list[str] = []
allowed_groups: list[str] = []
allowed_principals: list[str] = []
allowed_principal_labels: list[dict[str, Any]] = []
if document.acl:
try:
allowed_users = document.acl.allowed_users or []
allowed_groups = document.acl.allowed_groups or []
allowed_principals = document.acl.allowed_principals or []
allowed_principal_labels = document.acl.allowed_principal_labels or []
except AttributeError:
pass
# Prepare tweaks
connector_tweak_settings = None
if isinstance(self.ingest_settings, dict):
connector_tweak_settings = dict(self.ingest_settings)
connector_tweak_settings.pop("embeddingModel", None)
tweaks = self.connector_service.langflow_service.merge_ui_ingest_settings_into_tweaks(
{}, connector_tweak_settings
)
config = get_openrag_config()
effective_ingest_settings = (
dict(self.ingest_settings) if self.ingest_settings else {}
)
effective_ingest_settings["ocr"] = config.knowledge.ocr
effective_ingest_settings["pictureDescriptions"] = (
config.knowledge.picture_descriptions
)
effective_owner, effective_owner_name, effective_owner_email = (
resolve_shared_owner_fields(
self.user_id, self.owner_name, self.owner_email, shared
)
)
file_task.document_id = document.id
result = await self.connector_service.langflow_service.upload_and_ingest_file(
file_tuple=file_tuple,
session_id=None,
tweaks=tweaks,
settings=effective_ingest_settings,
jwt_token=self.jwt_token,
owner=effective_owner,
owner_name=effective_owner_name,
owner_email=effective_owner_email,
connector_type=connector_type,
docling_polling_service=self.connector_service.task_service.docling_polling_service
if self.connector_service.task_service
else None,
file_task=file_task,
document_id=document.id,
connector_file_id=document.id,
source_url=document.source_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,
allowed_principal_labels=allowed_principal_labels,
original_filename=file_task.filename,
original_mimetype=document.mimetype,
preview_mode=self.preview_mode,
upload_task_id=upload_task.task_id,
preview_user_id=self.user_id,
)
# Langflow returns "success" even when no text was extracted
# (e.g. image files without OCR). Verify the document actually
# landed in OpenSearch before declaring success.
# wait_for_visibility polls on an empty result to ride out
# OpenSearch's ~1s near-real-time window (the user-scoped
# client cannot force an indices:admin/refresh — it 403s).
if not await self.check_document_exists(
document.id,
_verification_client(opensearch_client),
on_error="assume_exists",
wait_for_visibility=True,
field="connector_file_id",
):
result = {
"status": "error",
"error": "No text content could be extracted from document",
}
# Persist connector metadata (incl. modified_time) onto the
# Langflow-indexed chunks (keyed by document_id) so bucket-connector
# change detection has a stored timestamp to compare against on the
# next sync. Mirrors the standard path's enrichment below.
if result.get("status") != "error":
await self.connector_service._update_connector_metadata(
document,
self.user_id,
connector_type,
self.jwt_token,
indexed_filename=file_task.filename,
)
else:
# Standard OpenRAG processing pipeline (process_document_standard)
standard_kwargs: dict[str, Any] = {}
if isinstance(self.ingest_settings, dict):
s = self.ingest_settings
em = s.get("embeddingModel")
if isinstance(em, str) and em.strip():
standard_kwargs["embedding_model"] = em.strip()
for ui_key, param in (
("chunkSize", "chunk_size"),
("chunkOverlap", "chunk_overlap"),
):
raw = s.get(ui_key)
if raw is not None:
try:
standard_kwargs[param] = int(raw)
except (TypeError, ValueError):
pass
config = get_openrag_config()
standard_kwargs["ocr"] = config.knowledge.ocr
standard_kwargs["picture_descriptions"] = config.knowledge.picture_descriptions
result = await self.process_document_standard(
file_path=tmp_path,
file_hash=file_hash,
owner_user_id=self.user_id,
original_filename=file_task.filename,
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=len(document.content),
connector_type=connector_type,
acl=document.acl,
connector_file_id=document.id,
shared=shared,
**standard_kwargs,
)
# Update indexed chunks with connector-specific metadata
if result["status"] in ["indexed", "unchanged"]:
await self.connector_service._update_connector_metadata(
document,
self.user_id,
connector_type,
self.jwt_token,
indexed_filename=file_task.filename,
)
# Add connector-specific metadata
result.update(
{
"source_url": document.source_url,
"document_id": document.id,
}
)
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
file_task.error = result.get("error") or "Failed to process document"
file_task.updated_at = time.time()
upload_task.failed_files += 1
else:
file_task.status = TaskStatus.COMPLETED
file_task.result = result
file_task.updated_at = time.time()
upload_task.successful_files += 1
except Exception as e:
file_task.status = TaskStatus.FAILED
file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
class S3FileProcessor(TaskProcessor):
"""Processor for files stored in S3 buckets"""
def __init__(
self,
document_service,
bucket: str,
s3_client=None,
owner_user_id: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
models_service=None,
docling_service=None,
replace_duplicates: bool = False,
*,
allow_anonymous_delete: bool,
):
import boto3
super().__init__(
document_service,
models_service,
docling_service,
)
self.bucket = bucket
self.s3_client = s3_client or boto3.client("s3")
self.owner_user_id = owner_user_id
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
self.replace_duplicates = replace_duplicates
# The uploading user's resolved knowledge:delete:anonymous; see
# delete_document_by_filename.
self.allow_anonymous_delete = allow_anonymous_delete
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
import time
from models.tasks import TaskStatus
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
try:
# The S3 key doubles as the indexed filename, so the duplicate
# gate can run before downloading the object.
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
self.owner_user_id, self.jwt_token
)
duplicate_action = await self.resolve_duplicate_filename(
item,
opensearch_client,
replace=self.replace_duplicates,
owner_user_id=self.owner_user_id,
claim_holder=self._claim_holder(upload_task, file_task),
allow_anonymous_delete=self.allow_anonymous_delete,
)
if duplicate_action in DUPLICATE_SKIP_ACTIONS:
self.mark_duplicate_skipped(upload_task, file_task)
return
suffix = os.path.splitext(item)[1]
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Download object to temporary file
with open(tmp_path, "wb") as tmp_file:
self.s3_client.download_fileobj(self.bucket, item, tmp_file)
# Compute hash
file_hash = hash_id(tmp_path)
# Get object size
try:
obj_info = self.s3_client.head_object(Bucket=self.bucket, Key=item)
file_size = obj_info.get("ContentLength", 0)
except Exception:
file_size = 0
# Use consolidated standard processing
result = await self.process_document_standard(
file_path=tmp_path,
file_hash=file_hash,
owner_user_id=self.owner_user_id,
original_filename=item, # Use S3 key as filename
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=file_size,
connector_type="s3",
)
result["path"] = f"s3://{self.bucket}/{item}"
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
file_task.error = result.get("error") or "Failed to process document"
upload_task.failed_files += 1
else:
file_task.status = TaskStatus.COMPLETED
file_task.result = result
upload_task.successful_files += 1
except Exception as e:
file_task.status = TaskStatus.FAILED
file_task.error = str(e) or repr(e)
upload_task.failed_files += 1
finally:
file_task.updated_at = time.time()
class LangflowFileProcessor(TaskProcessor):
"""Processor for Langflow file uploads with two-phase Docling + Langflow ingestion."""
def __init__(
self,
langflow_file_service,
session_manager,
owner_user_id: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
session_id: str = None,
tweaks: dict = None,
settings: dict = None,
replace_duplicates: bool = False,
connector_type: str = "local",
docling_polling_service=None,
preview_mode: bool = False,
*,
allow_anonymous_delete: bool,
):
super().__init__()
self.langflow_file_service = langflow_file_service
self.session_manager = session_manager
self.owner_user_id = owner_user_id
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
self.session_id = session_id
self.tweaks = tweaks or {}
self.settings = settings
self.replace_duplicates = replace_duplicates
self.connector_type = connector_type
self.docling_polling_service = docling_polling_service
self.preview_mode = preview_mode
# The uploading user's resolved knowledge:delete:anonymous; see
# delete_document_by_filename.
self.allow_anonymous_delete = allow_anonymous_delete
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a file path using LangflowFileService upload_and_ingest_file"""
# Update task status
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
try:
# Use the ORIGINAL filename stored in file_task (not the transformed temp path)
# This ensures we check/store the original filename with spaces, etc.
original_filename = file_task.filename or os.path.basename(item)
# Check if document with same filename already exists
opensearch_client = self.session_manager.get_user_opensearch_client(
self.owner_user_id, self.jwt_token
)
duplicate_action = await self.resolve_duplicate_filename(
original_filename,
opensearch_client,
replace=self.replace_duplicates,
owner_user_id=self.owner_user_id,
claim_holder=self._claim_holder(upload_task, file_task),
allow_anonymous_delete=self.allow_anonymous_delete,
)
if duplicate_action in DUPLICATE_SKIP_ACTIONS:
self.mark_duplicate_skipped(upload_task, file_task)
return
# Compute the content hash early — before reading file bytes into
# memory and before submitting any work to Docling so the duplicate
# guard fires as cheaply as possible.
#
# The guard only runs on the "proceed" path (no filename match).
# When duplicate_action == "replaced" the caller asked to replace an
# existing same-name document: the old chunks were already deleted
# above, so we must not short-circuit here even if the hash matches
# (that would leave the index empty after the delete).
file_hash = hash_id(item)
file_task.document_id = file_hash
if duplicate_action == "proceed" and await self.check_document_exists(
file_hash, opensearch_client
):
# Identical content is already indexed under a different filename.
# Report it as a warning rather than silently overwriting.
# Mirrors the connector path (processors.py:1212) but uses a
# distinct reason so the UI can distinguish it from a filename
# collision.
file_task.status = TaskStatus.SKIPPED
file_task.error = None
file_task.result = {
"status": "skipped",
"reason": "duplicate_content",
"warning": DUPLICATE_CONTENT_WARNING,
"document_id": file_hash,
}
file_task.updated_at = time.time()
upload_task.successful_files += 1
return
# Read file content for processing
with open(item, "rb") as f:
content = f.read()
# Create file tuple for upload using ORIGINAL filename
# This ensures the document is indexed with the original name
original_mimetype, _ = mimetypes.guess_type(original_filename)
if not original_mimetype:
original_mimetype = "application/octet-stream"
# Langflow's docling chokes on text/plain — rename .txt -> .md.
langflow_filename, content_type = langflow_safe_filename_and_mimetype(
original_filename, original_mimetype
)
file_tuple = (langflow_filename, content, content_type)
effective_jwt = self.jwt_token
if self.session_manager and not effective_jwt:
effective_jwt = self.session_manager.get_effective_jwt_token(
self.owner_user_id,
None,
)
# Prepare metadata tweaks similar to API endpoint
final_tweaks = self.tweaks.copy() if self.tweaks else {}
# Build settings with fresh OCR/pictureDescriptions from live
# config so retries pick up configuration changes.
config = get_openrag_config()
effective_settings = dict(self.settings) if self.settings else {}
effective_settings["ocr"] = config.knowledge.ocr
effective_settings["pictureDescriptions"] = config.knowledge.picture_descriptions
# Process file using langflow service. Passing the polling
# service triggers the two-phase model: backend polls Docling,
# then invokes Langflow only after SUCCESS. file_task is passed
# so phase / docling_status are tracked on the task record.
result = await self.langflow_file_service.upload_and_ingest_file(
file_tuple=file_tuple,
session_id=self.session_id,
tweaks=final_tweaks,
settings=effective_settings,
jwt_token=effective_jwt,
owner=self.owner_user_id,
owner_name=self.owner_name,
owner_email=self.owner_email,
connector_type=self.connector_type,
docling_polling_service=self.docling_polling_service,
file_task=file_task,
document_id=file_hash,
original_filename=original_filename,
original_mimetype=original_mimetype,
preview_mode=self.preview_mode,
upload_task_id=upload_task.task_id,
preview_user_id=self.owner_user_id,
)
# Langflow returns "success" even when no text was extracted
# (e.g. image files without OCR). Verify the document actually
# landed in OpenSearch before declaring success. We key off the
# filename — the identifier this path already uses for dedup and
# delete (see check_filename_exists / delete_document_by_filename
# above). The document_id (hash_id(item) == content hash) is now
# threaded through to Langflow so preview-mode index proof can look
# chunks up by document_id, but verification stays filename-based to
# match this path's existing dedup/delete semantics.
#
# wait_for_visibility polls on an empty result so the just-written
# chunks become visible within OpenSearch's near-real-time refresh
# window. We cannot force a refresh here: the user-scoped client
# lacks the indices:admin/refresh privilege (it 403s).
if not await self.check_filename_exists(
original_filename,
_verification_client(opensearch_client),
wait_for_visibility=True,
):
file_task.status = TaskStatus.FAILED
file_task.error = "No text content could be extracted from document"
file_task.updated_at = time.time()
upload_task.failed_files += 1
else:
# Update task with success
file_task.status = TaskStatus.COMPLETED
file_task.result = result
file_task.updated_at = time.time()
upload_task.successful_files += 1
except Exception as e:
# Update task with failure
file_task.status = TaskStatus.FAILED
file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise