OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Single deletion path for a connector file's indexed chunks.
Deletion propagation has multiple *triggers* — orphan reconciliation (file
missing from a full remote listing), per-file 404 cleanup during sync, rename
cleanup — but exactly one deletion *semantic*, owned here:
* Chunks are matched by the stable connector file id against BOTH id layouts:
the standard ingest path stores it in ``connector_file_id`` (``document_id``
holds the content hash), the Langflow path stores it in ``document_id``.
Matching a single field misses chunks from the other layout.
* Optionally owner-scoped: ``owner_user_id`` restricts to that owner's chunks;
``shared=True`` targets ownerless (instance-shared) chunks instead, and
``include_shared=True`` covers both (the right boundary when the caller has
already pinned the file down by its connector id — see
``build_owner_or_shared_filter``).
* DLS-safe: visible chunk ``_id``s are enumerated with the caller's
(user-scoped) client, then deleted individually with the trusted backend
write client — ``delete_by_query`` is silently no-opped under DLS.
"""
from collections.abc import Iterable
from typing import Any
def build_connector_file_chunks_query(
file_ids: Iterable[str],
*,
connector_type: str | None = None,
owner_user_id: str | None = None,
shared: bool = False,
include_shared: bool = False,
keep_filenames: Iterable[str] | None = None,
) -> dict[str, Any]:
"""Query matching every chunk of the given connector file ids.
``connector_type`` narrows the match to a single connector type so IDs
that happen to collide across connectors are not affected.
``include_shared`` widens ``owner_user_id`` from "owned by this user" to
"owned by this user OR ownerless". Sync paths need it: a file ingested with
the COS share-all toggle has no ``owner`` at all, so a plain owner term
matches none of its chunks and the cleanup silently no-ops. Ignored when
``shared=True`` (already ownerless-only) or when no owner is given.
``keep_filenames`` excludes chunks indexed under those names — used by
rename cleanup to drop only the stale old-name chunks of a file id.
"""
from utils.opensearch_queries import build_owner_or_shared_filter
ids = [fid for fid in file_ids if fid]
filters: list[dict[str, Any]] = [
{
"bool": {
"should": [
{"terms": {"document_id": ids}},
{"terms": {"connector_file_id": ids}},
# Some indices predate the explicit keyword mapping for
# connector_file_id, so it was dynamically mapped as
# analyzed text with a .keyword multi-field.
{"terms": {"connector_file_id.keyword": ids}},
],
"minimum_should_match": 1,
}
}
]
if connector_type:
filters.append({"term": {"connector_type": connector_type}})
if shared:
filters.append({"bool": {"must_not": {"exists": {"field": "owner"}}}})
elif owner_user_id:
filters.append(
build_owner_or_shared_filter(owner_user_id)
if include_shared
else {"term": {"owner": owner_user_id}}
)
query: dict[str, Any] = {"bool": {"filter": filters}}
keep = [name for name in (keep_filenames or []) if name]
if keep:
query["bool"]["must_not"] = [{"terms": {"filename": keep}}]
return query
async def delete_connector_file_chunks(
file_ids: Iterable[str],
opensearch_client,
*,
connector_type: str | None = None,
owner_user_id: str | None = None,
shared: bool = False,
include_shared: bool = False,
keep_filenames: Iterable[str] | None = None,
refresh: bool = False,
) -> int:
"""Delete the indexed chunks of the given connector file ids.
Returns the number of chunks deleted. Raises on infrastructure failure
(missing write client, OpenSearch errors) — callers that must never fail
their surrounding task wrap this in their own best-effort handling.
"""
from config.settings import clients, get_index_name
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
ids = [fid for fid in (file_ids or []) if fid]
if not ids:
return 0
write_client = clients.opensearch
if write_client is None:
raise RuntimeError("Backend OpenSearch write client is unavailable")
chunk_ids = await collect_visible_document_ids(
opensearch_client,
index=get_index_name(),
query=build_connector_file_chunks_query(
ids,
connector_type=connector_type,
owner_user_id=owner_user_id,
shared=shared,
include_shared=include_shared,
keep_filenames=keep_filenames,
),
)
return await delete_document_ids(
write_client,
index=get_index_name(),
document_ids=chunk_ids,
refresh=refresh,
)