OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import copy
from datetime import UTC, datetime
from typing import Any, NamedTuple
from fastapi import Depends, Request
from fastapi.responses import JSONResponse, PlainTextResponse
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from config.settings import get_index_name, is_workspace_oauth_overrides_enabled
from connectors.base import normalize_etag
from connectors.sharepoint.utils import is_valid_sharepoint_url
from dependencies import (
get_connector_service,
get_current_user,
get_db_session,
get_rbac_service,
get_session_manager,
has_effective_permission,
require_permission,
)
from services.connector_access_service import (
CONNECTOR_TYPES,
filter_connectors_for_user,
get_access_map,
is_connector_access_policy_enforced,
is_connector_allowed_for_request,
list_access_for_admin,
set_connector_access_bulk,
)
from session_manager import User
from utils.ingest_preview_flag import is_ingest_preview_enabled
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
# Maximum number of buckets returned by a single OpenSearch terms aggregation.
# Results are silently truncated when the true cardinality exceeds this value.
# See get_synced_file_ids_for_connector / get_synced_id_to_filename_map.
# TODO: replace with composite aggregation + pagination to handle arbitrary cardinality.
OPENSEARCH_TERMS_AGG_LIMIT = 10_000
async def _connector_access_denied(
request: Request,
session: AsyncSession,
connector_type: str,
) -> JSONResponse | None:
"""Return 403 when workspace policy blocks this connector type."""
if connector_type not in CONNECTOR_TYPES:
return None
if not is_connector_access_policy_enforced():
return None
if await is_connector_allowed_for_request(session, connector_type):
return None
return JSONResponse(
{"error": f"Connector not available: {connector_type}"},
status_code=403,
)
async def _allowed_connector_types_for_request(
request: Request,
session: AsyncSession,
connector_types: list[str],
) -> list[str]:
"""Drop connector types blocked by workspace policy (sync-all style endpoints)."""
if not is_connector_access_policy_enforced():
return connector_types
access_map = await get_access_map(session)
return [t for t in connector_types if access_map.get(t, True)]
def _connector_sync_should_replace(connector_type: str) -> bool:
"""Return True when sync should replace existing indexed files for this
connector type, so content changes propagate on re-sync.
Declared per connector via ``BaseConnector.SYNC_REPLACES_DUPLICATES``
(default True) instead of a hardcoded type list, so new connectors get
change-propagating sync without touching this module. Unknown types (e.g.
stale index docs from a removed enhancement connector) stay conservative
and skip duplicates.
"""
from connectors.registry import get_connector_class
cls = get_connector_class(connector_type)
return cls.SYNC_REPLACES_DUPLICATES if cls is not None else False
def _connector_uses_timestamp_change_detection(connector_type: str) -> bool:
"""True when sync should re-ingest only the files that actually differ at
source — by entity tag, or by a newer remote modified_time where there are
no tags to compare (see bucket_changed_file_ids) — instead of replacing
every indexed file.
Declared per connector via ``BaseConnector.CHANGE_DETECTION`` — the bucket
connectors set ``"timestamp"``; the default is ``"replace_always"``.
"""
from connectors.registry import get_connector_class
cls = get_connector_class(connector_type)
return cls is not None and cls.CHANGE_DETECTION == "timestamp"
def _is_unmapped_keyword_agg_error(err: Exception) -> bool:
"""True when a terms aggregation failed because the target field is an
analyzed `text` field without fielddata enabled — the error OpenSearch
raises for connector_file_id on indices that predate its addition to the
explicit `keyword` mapping in config/settings.py."""
msg = str(err)
return "Text fields are not optimised" in msg or "fielddata" in msg
async def get_synced_file_ids_for_connector(
connector_type: str,
user_id: str,
session_manager,
jwt_token: str = None,
) -> tuple[list[str], list[str], str]:
"""
Query OpenSearch for unique file IDs where connector_type matches.
Returns a 3-tuple ``(file_ids, filenames, id_field)``:
- ``file_ids``: connector source IDs to use for orphan detection and sync.
Comes from the ``connector_file_id`` field when chunks were indexed via
``ConnectorFileProcessor`` (non-Langflow path); falls back to ``document_id``
for Langflow-indexed chunks where ``document_id`` already holds the connector
source ID.
- ``filenames``: unique filenames as a fallback when ``file_ids`` is empty.
- ``id_field``: the OpenSearch field name that ``file_ids`` came from
(``"connector_file_id"`` or ``"document_id"``). Informational (logging);
chunk deletion matches both fields via ``connectors.chunk_cleanup``.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
query_body: dict[str, Any] = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
"unique_connector_file_ids": {
"terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}
},
"unique_document_ids": {
"terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT}
},
"unique_filenames": {
"terms": {"field": "filename", "size": OPENSEARCH_TERMS_AGG_LIMIT}
},
},
}
try:
result = await opensearch_client.search(index=get_index_name(), body=query_body)
except Exception as agg_err:
if not _is_unmapped_keyword_agg_error(agg_err):
raise
# Some indices predate connector_file_id's addition to the explicit
# mapping (config/settings.py), so it was dynamically mapped as
# analyzed text instead of keyword — terms aggs need the
# `.keyword` multi-field on those indices.
query_body["aggs"]["unique_connector_file_ids"]["terms"]["field"] = (
"connector_file_id.keyword"
)
result = await opensearch_client.search(index=get_index_name(), body=query_body)
# Prefer connector_file_id — these are set by ConnectorFileProcessor (non-Langflow)
# and hold the actual connector source IDs (e.g. SharePoint GUIDs), not SHA hashes.
connector_file_id_buckets = (
result.get("aggregations", {}).get("unique_connector_file_ids", {}).get("buckets", [])
)
connector_file_ids = [b["key"] for b in connector_file_id_buckets if b["key"]]
if len(connector_file_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
logger.warning(
"Connector file ID aggregation hit 10k limit - results may be truncated",
connector_type=connector_type,
returned_count=len(connector_file_ids),
)
if connector_file_ids:
file_ids = connector_file_ids
id_field = "connector_file_id"
else:
# Langflow path: document_id already holds the connector source ID.
doc_id_buckets = (
result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
)
file_ids = [b["key"] for b in doc_id_buckets if b["key"]]
if len(doc_id_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
logger.warning(
"Document ID aggregation hit 10k limit - results may be truncated",
connector_type=connector_type,
returned_count=len(file_ids),
)
id_field = "document_id"
filename_buckets = (
result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
)
filenames = [b["key"] for b in filename_buckets if b["key"]]
if len(filename_buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
logger.warning(
"Filename aggregation hit 10k limit - results may be truncated",
connector_type=connector_type,
returned_count=len(filenames),
)
logger.debug(
"Found synced files for connector",
connector_type=connector_type,
file_ids_count=len(file_ids),
id_field=id_field,
filenames_count=len(filenames),
)
return file_ids, filenames, id_field
except Exception as e:
logger.error(
"Failed to get synced file IDs",
connector_type=connector_type,
error=str(e),
)
return [], [], "document_id"
async def get_synced_id_to_filename_map(
connector_type: str,
user_id: str,
session_manager,
jwt_token: str | None = None,
) -> dict[str, str]:
"""Return an {id: filename} map for files ingested under this connector_type.
Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):
the non-Langflow path stores the connector id in ``connector_file_id`` (where
``document_id`` is a content hash), while the Langflow path stores it in
``document_id``. ``connector_file_id`` wins when both are present for the same id.
Uses a sub-aggregation so each id is paired with its top filename in a single
OpenSearch round trip.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
query_body: dict[str, Any] = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
"by_connector_file_id": {
"terms": {"field": "connector_file_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},
"aggs": {
"top_filename": {"terms": {"field": "filename", "size": 1}},
},
},
"by_document_id": {
"terms": {"field": "document_id", "size": OPENSEARCH_TERMS_AGG_LIMIT},
"aggs": {
"top_filename": {"terms": {"field": "filename", "size": 1}},
},
},
},
}
try:
result = await opensearch_client.search(index=get_index_name(), body=query_body)
except Exception as agg_err:
if not _is_unmapped_keyword_agg_error(agg_err):
raise
# See get_synced_file_ids_for_connector: some indices predate the
# explicit keyword mapping for connector_file_id.
query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (
"connector_file_id.keyword"
)
result = await opensearch_client.search(index=get_index_name(), body=query_body)
aggs = result.get("aggregations", {})
mapping: dict[str, str] = {}
# document_id first; connector_file_id overlays it (connector_file_id wins),
# matching get_synced_id_to_modified_time_map's precedence.
for agg_name in ("by_document_id", "by_connector_file_id"):
buckets = aggs.get(agg_name, {}).get("buckets", [])
if len(buckets) == OPENSEARCH_TERMS_AGG_LIMIT:
logger.warning(
"ID to filename mapping hit 10k limit - results may be truncated",
connector_type=connector_type,
agg=agg_name,
returned_count=len(buckets),
)
for bucket in buckets:
key = bucket.get("key")
if not key:
continue
fn_buckets = bucket.get("top_filename", {}).get("buckets", [])
mapping[key] = fn_buckets[0]["key"] if fn_buckets else ""
return mapping
except Exception as e:
logger.error(
"Failed to build id→filename map",
connector_type=connector_type,
error=str(e),
)
return {}
class SyncedFileState(NamedTuple):
"""What the index knows about an already-ingested connector file.
``modified_time_ms`` is the stored source timestamp; ``content_etag`` is the
object's stored entity tag. Either can be None for a file ingested before
that signal was recorded, or by a path that didn't persist it.
"""
modified_time_ms: float | None = None
content_etag: str | None = None
async def get_synced_file_state_map(
connector_type: str,
user_id: str,
session_manager,
jwt_token: str | None = None,
) -> dict[str, SyncedFileState]:
"""Map each ingested connector source id → the change-detection state stored for it.
Powers change detection for bucket connectors: callers compare a blob's
remote entity tag and ``modified_time`` against the stored ones to decide
whether a re-ingest is needed.
A key being **present** means the source id is already ingested under this
``connector_type``. Its fields may still be None — ingested before change
detection recorded that signal, or by a path that didn't enrich it — and
callers treat a file with no comparable signal at all as *unchanged*, to
avoid a mass re-ingest.
Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):
the non-Langflow path stores the connector id in ``connector_file_id`` (where
``document_id`` is a content hash), while the Langflow path stores it in
``document_id``. ``connector_file_id`` wins when both are present for the same id.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
def _sub_aggs(etag_field: str) -> dict[str, Any]:
return {
"latest_modified": {"max": {"field": "modified_time"}},
# All chunks of one file carry the same etag, so one bucket is enough.
"etag": {"terms": {"field": etag_field, "size": 1}},
}
def _build_body(id_field: str, etag_field: str) -> dict[str, Any]:
return {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
"by_connector_file_id": {
"terms": {"field": id_field, "size": 10000},
"aggs": _sub_aggs(etag_field),
},
"by_document_id": {
"terms": {"field": "document_id", "size": 10000},
"aggs": _sub_aggs(etag_field),
},
},
}
# See get_synced_file_ids_for_connector: some indices predate the explicit
# keyword mapping for these fields, so they were dynamically mapped as
# analyzed text with a `.keyword` multi-field and must be aggregated via
# that sub-field instead.
#
# A field may only be switched once a failure has PROVEN the drift: a
# terms agg on an analyzed text field raises, but one on a field that
# does not exist returns no buckets and no error. Switching speculatively
# therefore empties that aggregation silently.
#
# The two fields drift independently — connector_file_id predates the
# explicit mapping by far longer than content_etag, and an index can have
# either one stale — so each candidate switches only what the previous
# failure proved and leaves the other in the form that resolves. Retrying
# both together would silently drop every etag on a legacy index (whose
# content_etag, being newer, is correctly mapped and has no `.keyword`
# sub-field) or, worse, drop every connector source id from the map,
# leaving change detection to report every file as unchanged.
field_candidates = (
("connector_file_id", "content_etag"),
("connector_file_id.keyword", "content_etag"),
("connector_file_id", "content_etag.keyword"),
("connector_file_id.keyword", "content_etag.keyword"),
)
async def _search_through_mapping_drift() -> dict[str, Any]:
agg_err: Exception | None = None
for id_field, etag_field in field_candidates:
try:
return await opensearch_client.search(
index=get_index_name(), body=_build_body(id_field, etag_field)
)
except Exception as err:
if not _is_unmapped_keyword_agg_error(err):
raise
agg_err = err
raise agg_err # field_candidates is never empty
result = await _search_through_mapping_drift()
aggs = result.get("aggregations", {})
mapping: dict[str, SyncedFileState] = {}
# document_id first; connector_file_id overlays it (connector_file_id wins).
# The content-hash document_ids from the non-Langflow path are harmless noise —
# they never match an enumerated connector source id.
for agg_name in ("by_document_id", "by_connector_file_id"):
for bucket in aggs.get(agg_name, {}).get("buckets", []):
key = bucket.get("key")
if not key:
continue
etag_buckets = bucket.get("etag", {}).get("buckets", [])
mapping[key] = SyncedFileState(
modified_time_ms=bucket.get("latest_modified", {}).get("value"),
content_etag=normalize_etag(etag_buckets[0]["key"]) if etag_buckets else None,
)
return mapping
except Exception as e:
logger.error(
"Failed to build connector file state map",
connector_type=connector_type,
error=str(e),
)
return {}
# Tolerance (ms) to avoid a false-positive "changed" when a remote ISO timestamp
# round-trips through the OpenSearch ``date`` field with sub-second rounding.
_CHANGE_DETECTION_TOLERANCE_MS = 1000.0
def _parse_iso_to_epoch_ms(value: str | None) -> float | None:
"""Parse an ISO-8601 timestamp to epoch milliseconds, or None if unparseable."""
if not value:
return None
try:
s = value.strip()
if s.endswith("Z"):
s = s[:-1] + "+00:00"
dt = datetime.fromisoformat(s)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=UTC)
return dt.timestamp() * 1000.0
except (ValueError, TypeError):
return None
def remote_is_newer_than_synced(
file_id: str,
remote_modified_time: str | None,
synced_state_map: dict[str, SyncedFileState],
) -> bool:
"""True only when a stored ``modified_time`` exists and the remote is strictly newer.
Missing/unparseable timestamps (remote or stored) → False, so we never re-ingest
on ambiguity (backfill-safe).
"""
stored = synced_state_map.get(file_id)
stored_ms = stored.modified_time_ms if stored else None
if stored_ms is None:
return False
remote_ms = _parse_iso_to_epoch_ms(remote_modified_time)
if remote_ms is None:
return False
return remote_ms - stored_ms > _CHANGE_DETECTION_TOLERANCE_MS
def classify_remote_file_change(
file_id: str,
remote_modified_time: str | None,
is_ingested: bool,
synced_state_map: dict[str, SyncedFileState],
remote_etag: str | None = None,
) -> str:
"""Classify a remote blob as ``"new"`` / ``"changed"`` / ``"unchanged"``.
- ``new`` — not yet ingested.
- ``changed`` — ingested, and either the entity tags differ or (with no
tags to compare) the remote version is strictly newer.
- ``unchanged`` — ingested and provably identical, same-age/older, or the
change can't be proven at all (no stored signal, e.g. a
file ingested before change detection existed).
The entity tag is checked first and is decisive in both directions. Object
stores change it on every overwrite, so it catches a modification whose
timestamp we can't compare — and equal tags mean identical bytes, so a
timestamp that moved without the content changing (a copy, a re-upload of
the same file) no longer triggers a pointless re-ingest.
"""
if not is_ingested:
return "new"
stored = synced_state_map.get(file_id)
stored_etag = stored.content_etag if stored else None
remote_etag = normalize_etag(remote_etag)
if stored_etag and remote_etag:
return "unchanged" if stored_etag == remote_etag else "changed"
if remote_is_newer_than_synced(file_id, remote_modified_time, synced_state_map):
return "changed"
return "unchanged"
def has_comparable_change_signal(
file_id: str,
remote_modified_time: str | None,
synced_state_map: dict[str, SyncedFileState],
remote_etag: str | None = None,
) -> bool:
"""Whether a modification to this file could be detected at all.
False means no stored etag matching a remote one AND no stored timestamp to
compare — so the file is reported "unchanged" no matter what happened to it
at the source. Callers log the count: silently classifying an unknowable
file as up to date is exactly how a stale copy goes unnoticed.
"""
stored = synced_state_map.get(file_id)
if stored is None:
return False
if stored.content_etag and normalize_etag(remote_etag):
return True
return (
stored.modified_time_ms is not None
and _parse_iso_to_epoch_ms(remote_modified_time) is not None
)
async def bucket_changed_file_ids(
connector,
connector_type: str,
user_id: str,
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
) -> list[str]:
"""Return the already-ingested bucket file ids whose remote copy has changed.
Updates-only change detection for the Sync path: list the connector's blobs
once and keep the ids that are (a) already ingested and (b) differ at source
— by entity tag where both sides have one, otherwise by a strictly newer
``modified_time``. New blobs that aren't ingested yet are intentionally
ignored — Sync reconciles existing files; new files are added via the
connector's "Add from" panel. Listing exceptions propagate to the caller
(same as the bucket_filter sync path).
"""
existing_set = set(existing_file_ids)
if not existing_set:
return []
state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
changed_ids: list[str] = []
undetectable: list[str] = []
page_token = None
while True:
result = await connector.list_files(page_token=page_token)
for f in result.get("files", []):
fid = f.get("id")
if not fid or fid not in existing_set:
continue
remote_modified_time = f.get("modified_time")
remote_etag = f.get("etag")
if (
classify_remote_file_change(fid, remote_modified_time, True, state_map, remote_etag)
== "changed"
):
changed_ids.append(fid)
elif not has_comparable_change_signal(
fid, remote_modified_time, state_map, remote_etag
):
undetectable.append(fid)
page_token = result.get("next_page_token")
if not page_token:
break
if undetectable:
# These report "unchanged" whatever happened to them at the source —
# nothing stored to compare against. Say so, rather than letting a stale
# copy look up to date. Re-ingesting them once (via the connector's
# "Add from" panel, overwriting) records a signal and fixes it for good.
logger.warning(
"Files have no stored change-detection signal and cannot be checked for updates",
connector_type=connector_type,
count=len(undetectable),
sample=undetectable[:5],
)
return changed_ids
async def list_remote_files_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
existing_file_ids: list[str],
) -> dict[str, dict[str, Any]] | None:
"""Enumerate what currently exists at the source, as {file_id: record}.
One listing pass shared by everything that needs to diff the index against
the source — orphan detection and sync preview both read it, so a preview
doesn't enumerate the whole bucket twice.
Returns None (not {}) when strict gating aborts the pass: no active
connection, an unauthenticated one, or a listing that raised. Callers must
treat that as "could not determine safely" rather than "nothing there" —
the difference between leaving the index alone and deleting all of it.
"""
connections = await connector_service.connection_manager.list_connections(
user_id=user_id, connector_type=connector_type
)
active = [c for c in connections if c.is_active]
if not active:
logger.info(
"Skipping remote listing — no active connections",
connector_type=connector_type,
)
return None
remote_files: dict[str, dict[str, Any]] = {}
for conn in active:
try:
connector = await connector_service.get_connector(conn.connection_id)
if not connector or not connector.is_authenticated:
logger.info(
"Skipping remote listing — connection unauthenticated",
connector_type=connector_type,
connection_id=conn.connection_id,
)
return None
# Drive the per-id existence check via list_selected_files when
# the connector supports it (SharePoint / OneDrive / Google Drive).
# The flat default of list_files() only returns the *root* listing
# (e.g. /drive/root/children for SharePoint, files-only, no folder
# traversal), so any folder-internal file in OpenSearch would be
# absent from the result and wrongly flagged as an orphan.
# list_selected_files iterates each id via _get_file_metadata_by_id
# and silently drops missing ids, so the result is exactly "the
# subset of existing_file_ids that still exists at source" — which
# is what orphan detection actually needs.
# cfg is None on bucket connectors (BaseConnector declares it as a
# class default), so guard on cfg-is-not-None rather than hasattr:
# otherwise bucket connectors route through list_selected_files ->
# list_files() (the whole account) instead of the flat listing below.
scoped_listing = getattr(connector, "cfg", None) is not None and bool(existing_file_ids)
if scoped_listing:
page = await connector.list_selected_files(list(existing_file_ids))
for f in page.get("files", []):
fid = f.get("id")
if fid:
remote_files[fid] = f
else:
page_token = None
while True:
page = await connector.list_files(page_token=page_token)
for f in page.get("files", []):
fid = f.get("id")
if fid:
remote_files[fid] = f
page_token = page.get("nextPageToken") or page.get("next_page_token")
if not page_token:
break
except Exception as e:
logger.warning(
"Skipping remote listing — listing failed",
connector_type=connector_type,
connection_id=conn.connection_id,
error=str(e),
)
return None
return remote_files
async def compute_orphans_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
id_to_filename: dict[str, str] | None = None,
remote_files: dict[str, dict[str, Any]] | None = None,
) -> list[dict[str, str]] | None:
"""Compute orphan documents (ingested but no longer present at the source)
for this connector_type without deleting them.
Returns a list of {"document_id", "filename"} dicts. Returns None when strict
gating aborts the pass (unauthenticated connection or listing exception) so
callers can distinguish "no orphans" from "could not determine safely".
``remote_files`` lets a caller that has already enumerated the source pass
the listing in instead of paying for a second pass.
"""
if not existing_file_ids:
return []
if remote_files is None:
remote_files = await list_remote_files_for_connector_type(
connector_type=connector_type,
user_id=user_id,
connector_service=connector_service,
existing_file_ids=existing_file_ids,
)
if remote_files is None:
return None
orphan_ids = [fid for fid in existing_file_ids if fid not in remote_files]
if not orphan_ids:
return []
fn_map = id_to_filename or {}
return [{"document_id": fid, "filename": fn_map.get(fid, "")} for fid in orphan_ids]
async def delete_orphan_documents(
orphan_ids: list[str],
user_id: str,
session_manager,
jwt_token: str | None,
*,
connector_type: str | None = None,
shared: bool | None = False,
allow_anonymous_delete: bool = False,
) -> int:
"""Delete OpenSearch chunks for the given orphan IDs. Returns the number of
chunks deleted (0 on failure).
Deletion matches both id layouts (``connector_file_id`` for the standard
ingest path, ``document_id`` for the Langflow path) via
``connectors.chunk_cleanup``, so callers no longer need to track which
field the ids came from.
``shared`` is the caller's explicit intent and is None on the re-sync paths,
which have no record of how each file was ingested.
``allow_anonymous_delete`` is the resolved ``knowledge:delete:anonymous``
permission and is what puts ownerless (share-all) chunks in scope at all.
Without it this stays owner-scoped, even on a re-sync that would otherwise
widen to "owned by me OR ownerless".
That gate is not paranoia about the caller's own files. The orphan id set
comes from ``get_synced_file_ids_for_connector``, which filters only on
``connector_type`` and reads through the user-scoped client — and DLS makes
every ownerless chunk visible to every authenticated user
(securityconfig/roles.yml). So the set can include shared files ingested by
a *different* user's connection, which this user's own remote listing will
not contain and orphan detection will therefore classify as deleted at
source. Widening the delete scope without the permission would let any
authenticated user erase another connection's shared documents by running
an ordinary sync.
"""
if not orphan_ids:
return 0
from connectors.chunk_cleanup import delete_connector_file_chunks
include_shared = allow_anonymous_delete and not shared
if shared and not allow_anonymous_delete:
logger.warning(
"Skipping shared orphan cleanup — caller lacks knowledge:delete:anonymous",
connector_type=connector_type,
orphan_count=len(orphan_ids),
)
return 0
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
return await delete_connector_file_chunks(
orphan_ids,
opensearch_client,
connector_type=connector_type,
owner_user_id=None if shared else user_id,
shared=bool(shared),
include_shared=include_shared,
refresh=True,
)
except Exception as e:
logger.error(
"Orphan delete failed",
orphan_count=len(orphan_ids),
error=str(e),
)
return 0
async def reconcile_orphans_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
*,
shared: bool | None = False,
allow_anonymous_delete: bool = False,
) -> list[str]:
"""Compute and delete orphans for a connector type. Thin wrapper around
compute_orphans_for_connector_type + delete_orphan_documents preserved for
callers that perform sync immediately after reconcile.
``allow_anonymous_delete`` (the resolved ``knowledge:delete:anonymous``
permission) is forwarded to the deletion; see delete_orphan_documents for
why ownerless chunks must not be in scope without it.
Returns the list of orphan file IDs that were deleted (or []).
"""
orphans = await compute_orphans_for_connector_type(
connector_type=connector_type,
user_id=user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
)
if not orphans:
return []
orphan_ids = [o["document_id"] for o in orphans]
deleted = await delete_orphan_documents(
allow_anonymous_delete=allow_anonymous_delete,
orphan_ids=orphan_ids,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
connector_type=connector_type,
shared=shared,
)
logger.info(
"Orphan reconcile complete",
connector_type=connector_type,
orphan_count=len(orphan_ids),
deleted_chunks=deleted,
)
if deleted <= 0:
return []
return orphan_ids
async def _sync_existing_connector_files(
connector_type: str,
working_connection,
user_id: str,
connector_service,
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
existing_filenames: list[str],
id_field: str,
*,
ingest_settings: dict[str, Any] | None = None,
shared: bool | None = None,
allow_anonymous_delete: bool = False,
reconcile: bool = True,
max_files: int | None = None,
) -> dict[str, Any]:
"""Re-sync the files already indexed for a connector type — the shared
no-selection Sync flow used by both ``connector_sync`` and
``sync_all_connectors``.
Orphans (deleted at the source) are reconciled first when ``reconcile`` is
True, then either timestamp change detection (updates-only re-ingest) or a
full re-sync runs depending on the connector's ``CHANGE_DETECTION``
capability; connectors with no stored ids fall back to filename filtering.
``shared`` defaults to None — "inherit". This flow re-syncs files that are
already indexed, and nothing records whether each was ingested with the COS
share-all toggle, so each file's current indexed state decides (see
``ConnectorFileProcessor._resolve_shared``). Passing False here instead
would make sync treat every shared COS file as private, skip it as an
unreplaceable duplicate, and leave the stale content in the index.
Returns an outcome dict the caller maps onto its own response shape:
* ``{"outcome": "synced", "task_id": ...}``
* ``{"outcome": "deleted_only"}`` — orphan cleanup removed every file;
nothing left to sync.
* ``{"outcome": "up_to_date"}`` — timestamp change detection found no
remote changes.
"""
if existing_file_ids:
logger.info(
"Syncing specific files by connector file ID",
connector_type=connector_type,
file_count=len(existing_file_ids),
id_field=id_field,
)
# Reconcile orphans (files deleted at the source) before re-syncing.
# Callers gate this: a capped sync sees a partial remote listing and
# would delete legitimate files.
ids_to_sync = list(existing_file_ids)
if reconcile:
orphan_ids = await reconcile_orphans_for_connector_type(
connector_type=connector_type,
user_id=user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
if orphan_ids:
orphan_id_set = set(orphan_ids)
ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]
if not ids_to_sync:
return {"outcome": "deleted_only"}
if _connector_uses_timestamp_change_detection(connector_type):
# Timestamp Sync is updates-only: re-ingest just the files whose
# remote copy is newer than what's indexed (deleting the stale
# chunks via replace_duplicates). Unlike replace_duplicates=False
# this actually propagates content changes; unlike replacing every
# id it skips unchanged files instead of re-fetching the source.
connector = await connector_service.get_connector(working_connection.connection_id)
changed_ids = await bucket_changed_file_ids(
connector,
connector_type,
user_id,
session_manager,
jwt_token,
ids_to_sync,
)
if not changed_ids:
return {"outcome": "up_to_date"}
if max_files is not None:
changed_ids = changed_ids[:max_files]
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user_id,
changed_ids,
jwt_token=jwt_token,
ingest_settings=ingest_settings,
replace_duplicates=True,
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
else:
if max_files is not None:
ids_to_sync = ids_to_sync[:max_files]
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user_id,
ids_to_sync,
jwt_token=jwt_token,
ingest_settings=ingest_settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
else:
# Fallback: use filename filtering (for Langflow-ingested files without document_id)
logger.info(
"Syncing files by filename filter (document_id not available)",
connector_type=connector_type,
filename_count=len(existing_filenames),
)
task_id = await connector_service.sync_connector_files(
working_connection.connection_id,
user_id,
max_files=max_files,
jwt_token=jwt_token,
filename_filter=set(existing_filenames),
ingest_settings=ingest_settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
return {"outcome": "synced", "task_id": task_id}
class ConnectorSyncBody(BaseModel):
# The connection this sync targets. The upload UI is rendered per connection
# and sends it on every request; connector_check_duplicates resolves the same
# id, so honouring it here keeps the dialog and the sync it confirms pointed
# at one connection. Unset (or unknown) falls back to the first that
# authenticates, as before.
connection_id: str | None = None
max_files: int | None = None
selected_files: list[Any] | None = None
# When True, ingest ALL files from the connector (bypasses the existing-files gate).
# Used by bucket-kind connectors on initial ingest.
sync_all: bool = False
# When set, only ingest files from these buckets (bucket-kind connectors).
bucket_filter: list[str] | None = None
# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).
settings: dict[str, Any] | None = None
# When True, files whose filename already exists in the index are replaced
# rather than failing. Set by the provider upload UI after the user confirms
# overwrite in the duplicate dialog. Honored for an explicit ``selected_files``
# sync and for a whole-container ``bucket_filter`` sync, where it also
# bypasses the modified-time/etag gate (see connector_sync).
replace_duplicates: bool = False
# When True (OSS only for now; SaaS deferred), run the ingest in preview mode
# (same as direct upload). Honored only when is_ingest_preview_enabled().
preview: bool = False
# When True (COS only), index chunks without an owner field so OpenSearch DLS
# makes them visible to all users in the instance. Temporary CIO mechanism;
# not a full ACL feature.
# Omitted/None means "no explicit intent": new files are indexed private,
# and files already in the index keep the sharing state they have. The
# connector UI already sends the flag only when the toggle is on screen.
shared: bool | None = None
class ConnectorCheckDuplicatesBody(BaseModel):
connection_id: str | None = None
selected_files: list[Any] | None = None
# Bucket-kind connectors (aws_s3, azure_blob, ibm_cos) select whole
# buckets rather than individual files; when set (and selected_files is
# not), the check lists files from these buckets and classifies each as
# new/changed/unchanged instead of a plain filename match.
bucket_filter: list[str] | None = None
def _connector_file_response(file_info: dict[str, Any], cleaned_name: str | None = None) -> dict:
"""Normalize connector file metadata into the upload page's CloudFile shape."""
response = {
"id": file_info.get("id"),
"name": cleaned_name or file_info.get("name", ""),
"mimeType": file_info.get("mimeType")
or file_info.get("mime_type")
or file_info.get("mimetype")
or "",
"isFolder": bool(file_info.get("isFolder", False)),
}
for source_key, target_key in (
("size", "size"),
("webUrl", "webUrl"),
("url", "webUrl"),
("webViewLink", "webViewLink"),
("downloadUrl", "downloadUrl"),
("download_url", "downloadUrl"),
):
value = file_info.get(source_key)
if value is not None and value != "":
response[target_key] = value
return response
async def _expand_selected_connector_files(
connector,
selected_files_raw: list[Any],
) -> list[dict[str, Any]]:
file_ids = [f.get("id") for f in selected_files_raw if isinstance(f, dict) and f.get("id")]
expanded_files_info: list[dict[str, Any]] = []
# cfg is None on bucket connectors (BaseConnector class default), so guard on
# cfg-is-not-None: only cfg-backed connectors expand folders here; bucket
# connectors fall through to using selected_files_raw directly below.
if file_ids and getattr(connector, "cfg", None) is not None:
try:
result = await connector.list_selected_files(file_ids)
for f in result.get("files", []):
expanded_files_info.append(_connector_file_response(f))
except Exception as e:
logger.error("Failed to expand files in duplicate check", error=str(e))
if not expanded_files_info:
for f in selected_files_raw:
if isinstance(f, dict) and not f.get("isFolder"):
expanded_files_info.append(_connector_file_response(f))
return expanded_files_info
async def _classify_connector_duplicates(
connector,
connector_type: str,
selected_files_raw: list[Any],
session_manager,
user_id: str,
jwt_token: str | None,
) -> dict[str, Any]:
"""Expand connector selections and split them into duplicate/non-duplicate files."""
expanded_files_info = await _expand_selected_connector_files(connector, selected_files_raw)
if not expanded_files_info:
# The selection named only folders that expanded to nothing, so there is
# nothing to compare. Counted against what the caller asked for, which
# separates an empty expansion from an empty request.
logger.info(
"Duplicate check found nothing to check",
connector_type=connector_type,
selected=len(selected_files_raw),
)
return {
"duplicate_names": [],
"duplicate_files": [],
"non_duplicate_files": [],
"duplicate_count": 0,
"total_files": 0,
}
from utils.file_utils import clean_connector_filename, get_filename_aliases
cleaned_files = []
all_candidates = set()
for file_info in expanded_files_info:
cleaned_name = clean_connector_filename(file_info["name"], file_info["mimeType"])
response_file = _connector_file_response(file_info, cleaned_name=cleaned_name)
aliases = get_filename_aliases(cleaned_name)
cleaned_files.append((response_file, aliases))
all_candidates.update(aliases)
if not all_candidates:
# Files expanded but none yielded a comparable name, so the index was
# never queried: every file is reported new without having been checked.
logger.info(
"Duplicate check skipped: no comparable filenames",
connector_type=connector_type,
listed=len(cleaned_files),
)
return {
"duplicate_names": [],
"duplicate_files": [],
"non_duplicate_files": [file_info for file_info, _ in cleaned_files],
"duplicate_count": 0,
"total_files": len(cleaned_files),
}
from utils.opensearch_filenames import find_existing_filenames
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
existing_filenames = set()
try:
existing_filenames = await find_existing_filenames(
all_candidates, opensearch_client, get_index_name()
)
except Exception as search_err:
if "index_not_found_exception" not in str(search_err):
raise
duplicate_files = []
non_duplicate_files = []
duplicate_names = []
for file_info, aliases in cleaned_files:
if any(alias in existing_filenames for alias in aliases):
duplicate_files.append(file_info)
duplicate_names.append(file_info["name"])
else:
non_duplicate_files.append(file_info)
# See the bucket classifier: logged so a "no dialog appeared" report can be
# answered from the server side.
logger.info(
"Duplicate check complete",
connector_type=connector_type,
selected=len(cleaned_files),
duplicates=len(duplicate_files),
new_files=len(non_duplicate_files),
)
return {
"duplicate_names": list(dict.fromkeys(duplicate_names)),
"duplicate_files": duplicate_files,
"non_duplicate_files": non_duplicate_files,
"duplicate_count": len(duplicate_files),
"total_files": len(cleaned_files),
}
def _connector_scoped_to_buckets(connector, bucket_names: list[str]):
"""Shallow copy of a (cached, shared) connector with bucket_names overridden.
connector_service.get_connector() returns a per-connection cached instance;
mutating bucket_names on it directly would leak the override into concurrent
requests using the same connection.
"""
scoped = copy.copy(connector)
scoped.bucket_names = list(bucket_names)
return scoped
def _cleaned_blob_filename(file_info: dict[str, Any]) -> str:
"""The name a remote blob would be indexed under, cleaned exactly as
ingestion cleans it — so name comparisons here mean what they say."""
from utils.file_utils import clean_connector_filename
return clean_connector_filename(
file_info.get("name", ""),
file_info.get("mimeType") or file_info.get("mime_type") or file_info.get("mimetype") or "",
)
# Ids travel to OpenSearch in `terms` clauses, which are capped by
# index.max_terms_count (65_536 by default). Whole-container syncs can exceed
# that, so id lookups go out in batches.
_ID_LOOKUP_BATCH_SIZE = 1024
async def _bucket_blob_ids_already_synced(
file_ids: list[str],
connector_type: str,
session_manager,
user_id: str,
jwt_token: str | None,
) -> set[str]:
"""Which of exactly these blob ids already have chunks under this connector.
``get_synced_file_ids_for_connector`` answers the same question for the
whole connector in one terms aggregation, which is capped at
``OPENSEARCH_TERMS_AGG_LIMIT``: past the cap it omits ids that ARE ingested.
Reading "absent from that set" as "this connector never synced it" would
then turn an already-synced blob into a name collision — its own indexed
document is what holds the name — and re-ingest it under overwrite, in
exactly the large containers where the cap is reached. Asking about the
handful of ids actually in question keeps the answer independent of how much
the connector has ingested in total.
The id is matched in both fields it can live in, as
``connectors.chunk_cleanup`` does: the standard ingest path stores it in
``connector_file_id``, the Langflow path in ``document_id``.
"""
ids = [fid for fid in file_ids if fid]
if not ids:
return set()
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
synced: set[str] = set()
for start in range(0, len(ids), _ID_LOOKUP_BATCH_SIZE):
batch = ids[start : start + _ID_LOOKUP_BATCH_SIZE]
body: dict[str, Any] = {
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"connector_type": connector_type}},
{
"bool": {
"should": [
{"terms": {"document_id": batch}},
{"terms": {"connector_file_id": batch}},
# Indices predating the explicit keyword
# mapping dynamically mapped this as text.
{"terms": {"connector_file_id.keyword": batch}},
],
"minimum_should_match": 1,
}
},
]
}
},
"aggs": {
"connector_file_ids": {"terms": {"field": "connector_file_id", "size": len(batch)}},
"document_ids": {"terms": {"field": "document_id", "size": len(batch)}},
},
}
try:
response = await opensearch_client.search(index=get_index_name(), body=body)
except Exception as search_err:
if _is_unmapped_keyword_agg_error(search_err):
body["aggs"]["connector_file_ids"]["terms"]["field"] = "connector_file_id.keyword"
response = await opensearch_client.search(index=get_index_name(), body=body)
elif "index_not_found_exception" in str(search_err):
return set()
else:
raise
aggregations = response.get("aggregations", {})
for agg_name in ("connector_file_ids", "document_ids"):
for bucket in aggregations.get(agg_name, {}).get("buckets", []):
if bucket.get("key"):
synced.add(bucket["key"])
# Only ids we asked about; the aggregations can also surface an id that
# merely shares a chunk with one of them.
return synced & set(ids)
async def _bucket_blob_ids_with_indexed_filename(
files: list[dict[str, Any]],
session_manager,
user_id: str,
jwt_token: str | None,
) -> dict[str, str]:
"""Map each blob whose filename already has indexed chunks — from any source,
a direct upload, another connector, or an earlier sync of this one — to the
indexed name it lands on.
The name, not just the id, because two blobs in one container can land on
the same one: a bucket connector names a blob after its key's basename, so
``a/report.pdf`` and ``b/report.pdf`` are both ``report.pdf``. Callers that
act on the collision need to know which of them compete.
One batched terms aggregation for the whole listing, through the same
``find_existing_filenames`` the OAuth connector classifier and the per-file
processor backstop use, so the duplicate question gets one answer at every
altitude. A missing index means nothing is indexed yet, not a failure.
"""
from utils.file_utils import get_filename_aliases
from utils.opensearch_filenames import find_existing_filenames
aliases_by_id: dict[str, list[str]] = {}
all_candidates: set[str] = set()
for f in files:
fid = f.get("id")
if not fid:
continue
aliases = get_filename_aliases(_cleaned_blob_filename(f))
aliases_by_id[fid] = aliases
all_candidates.update(aliases)
if not all_candidates:
return {}
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
try:
existing_filenames = await find_existing_filenames(
all_candidates, opensearch_client, get_index_name()
)
except Exception as search_err:
if "index_not_found_exception" not in str(search_err):
raise
return {}
# Listing order, so a caller resolving a contested name gets a stable winner
# (bucket listings are ordered by key).
matched: dict[str, str] = {}
for fid, aliases in aliases_by_id.items():
for alias in aliases:
if alias in existing_filenames:
matched[fid] = alias
break
return matched
async def _classify_bucket_connector_duplicates(
connector,
connector_type: str,
bucket_filter: list[str],
session_manager,
user_id: str,
jwt_token: str | None,
) -> dict[str, Any]:
"""Preview a bucket_filter sync: classify remote blobs as duplicate or new
without ingesting anything, so the upload UI can confirm an overwrite.
A blob is a duplicate when it is already indexed under THIS connector type
(id match) or when its filename is already in the index from any other
source — a direct upload, a different connector. Blobs that match neither
are returned as ``non_duplicate_files`` so the caller can sync just those
when the user chooses to skip duplicates.
"""
scoped_connector = _connector_scoped_to_buckets(connector, bucket_filter)
all_files: list[dict[str, Any]] = []
page_token = None
while True:
result = await scoped_connector.list_files(page_token=page_token)
all_files.extend(result.get("files", []))
page_token = result.get("next_page_token")
if not page_token:
break
if not all_files:
logger.info(
"Duplicate check found nothing to check",
connector_type=connector_type,
buckets=bucket_filter,
)
return {
"duplicate_names": [],
"duplicate_files": [],
"non_duplicate_files": [],
"duplicate_count": 0,
"total_files": 0,
}
existing_ids, _, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
existing_set = set(existing_ids)
# The id half answers "did this connector already ingest this blob?"; the
# filename half answers "is this name already in the index at all?" — the
# question a direct upload, or an ingest through a different connector,
# makes relevant. Only the id half existed here, so a bucket whose blobs
# collide by name with files ingested elsewhere reported zero duplicates,
# skipped the confirm dialog, and then had every colliding file skipped at
# ingest time with "A file with this name already exists".
name_taken_ids = await _bucket_blob_ids_with_indexed_filename(
all_files, session_manager, user_id, jwt_token
)
duplicate_files: list[dict[str, Any]] = []
duplicate_names: list[str] = []
non_duplicate_files: list[dict[str, Any]] = []
total = 0
already_synced = 0
for f in all_files:
fid = f.get("id")
if not fid:
continue
total += 1
response_file = _connector_file_response(f, cleaned_name=_cleaned_blob_filename(f))
id_match = fid in existing_set
if id_match:
already_synced += 1
if id_match or fid in name_taken_ids:
duplicate_files.append(response_file)
duplicate_names.append(response_file["name"])
else:
non_duplicate_files.append(response_file)
# Both halves of the answer, so a "the dialog never appeared" report can be
# settled from the logs: whether the listing came back, whether the names
# were found in the index, and what the caller was told. Without it the only
# way to tell a check that found nothing from one that never ran is the
# browser's network tab.
logger.info(
"Duplicate check complete",
connector_type=connector_type,
buckets=bucket_filter,
listed=total,
already_synced_here=already_synced,
name_taken_elsewhere=len(set(name_taken_ids) - existing_set),
duplicates=len(duplicate_files),
new_files=len(non_duplicate_files),
)
return {
"duplicate_names": list(dict.fromkeys(duplicate_names)),
"duplicate_files": duplicate_files,
"non_duplicate_files": non_duplicate_files,
"duplicate_count": len(duplicate_files),
"total_files": total,
}
async def connector_check_duplicates(
connector_type: str,
body: ConnectorCheckDuplicatesBody,
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
):
"""Check if any of the selected files or folders contain files that already exist in the index"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
selected_files_raw = body.selected_files
if not selected_files_raw and not body.bucket_filter:
# Returning an empty duplicate list here reads to the caller as "checked,
# found nothing", and the UI goes on to ingest without ever offering the
# overwrite choice — the same thing a broken check looks like. The
# request asked about nothing, so say that instead of answering it.
logger.warning(
"Duplicate check asked about nothing",
connector_type=connector_type,
connection_id=body.connection_id,
)
return JSONResponse(
{"error": "Specify selected_files or bucket_filter to check for duplicates"},
status_code=400,
)
try:
jwt_token = user.jwt_token
# Get all active connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
)
active_connections = [conn for conn in connections if conn.is_active]
# If connection_id is provided, find it, otherwise find the first working connection
working_connection = None
if body.connection_id:
for conn in active_connections:
if conn.connection_id == body.connection_id:
working_connection = conn
break
if not working_connection:
for conn in active_connections:
try:
connector = await connector_service.get_connector(conn.connection_id)
if connector and await connector.authenticate():
working_connection = conn
break
except Exception:
continue
if not working_connection:
return JSONResponse(
{"error": f"No working {connector_type} connections found"},
status_code=404,
)
connector = await connector_service.get_connector(working_connection.connection_id)
if not connector:
return JSONResponse(
{"error": f"Connection '{working_connection.connection_id}' not found"},
status_code=404,
)
if body.bucket_filter and not selected_files_raw:
return JSONResponse(
await _classify_bucket_connector_duplicates(
connector=connector,
connector_type=connector_type,
bucket_filter=body.bucket_filter,
session_manager=session_manager,
user_id=user.user_id,
jwt_token=jwt_token,
)
)
return JSONResponse(
await _classify_connector_duplicates(
connector=connector,
connector_type=connector_type,
selected_files_raw=selected_files_raw,
session_manager=session_manager,
user_id=user.user_id,
jwt_token=jwt_token,
)
)
except Exception:
logger.exception("[CONNECTOR] Error checking duplicates")
return JSONResponse({"error": "An internal error has occurred."}, status_code=500)
async def list_connectors(
request: Request,
connector_service=Depends(get_connector_service),
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
):
"""List available connector types with metadata"""
try:
connector_types = connector_service.connection_manager.get_available_connector_types(
user_id=user.user_id
)
if is_connector_access_policy_enforced():
access_map = await get_access_map(session)
connector_types = filter_connectors_for_user(connector_types, access_map)
return JSONResponse({"connectors": connector_types})
except Exception as e:
logger.error("[CONNECTOR] Error listing connectors", error=str(e))
return JSONResponse({"connectors": []})
class UpdateConnectorAccessBody(BaseModel):
access: dict[str, bool]
def _connector_access_client_error(exc: ValueError) -> str:
"""Safe client message for set_connector_access_bulk validation failures."""
detail = str(exc)
if detail.startswith("Unknown connector type:"):
return "Unknown connector type"
return "Invalid request data"
async def get_connector_workspace_policy(
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
):
"""Read-only stored workspace connector overrides for Connectors tab filtering.
Returns only explicit admin saves (missing types default to allowed server-side
but still follow deployment visibility rules on the client).
"""
from db.repositories import WorkspaceConfigRepo
from services.connector_access_service import CONNECTOR_ACCESS_SECTION
stored = await WorkspaceConfigRepo(session).get_section(CONNECTOR_ACCESS_SECTION) or {}
return JSONResponse({"access": stored})
async def get_connector_user_access(
connector_service=Depends(get_connector_service),
user: User = Depends(require_permission("connectors:manage:access")),
session: AsyncSession = Depends(get_db_session),
):
"""List connector types and whether they are enabled for this workspace."""
metadata = connector_service.connection_manager.get_available_connector_types(
user_id=user.user_id
)
connectors = await list_access_for_admin(session, metadata)
return JSONResponse({"connectors": connectors})
async def update_connector_user_access(
body: UpdateConnectorAccessBody,
user: User = Depends(require_permission("connectors:manage:access")),
session: AsyncSession = Depends(get_db_session),
connector_service=Depends(get_connector_service),
):
"""Save workspace connector availability policy."""
try:
await set_connector_access_bulk(
session,
body.access,
user.db_user_id or user.user_id,
)
await session.commit()
except ValueError as e:
logger.error(
"[CONNECTOR] Invalid connector access update",
error=str(e),
)
return JSONResponse(
{"error": _connector_access_client_error(e)},
status_code=400,
)
metadata = connector_service.connection_manager.get_available_connector_types(
user_id=user.user_id
)
connectors = await list_access_for_admin(session, metadata)
return JSONResponse({"connectors": connectors})
class UpdateConnectorOAuthConfigBody(BaseModel):
client_id: str | None = None
client_secret: str | None = None
def _oauth_config_unavailable_response() -> JSONResponse:
return JSONResponse(
{"error": "Workspace OAuth connector credential overrides are not enabled"},
status_code=404,
)
async def get_connector_oauth_config(
user: User = Depends(require_permission("connectors:manage:access")),
session: AsyncSession = Depends(get_db_session),
):
"""Per OAuth-kind connector: whether a workspace credential override is set,
plus env-var fallback visibility. Never returns the decrypted secret."""
from services.connector_oauth_config_service import get_oauth_config_status
if not is_workspace_oauth_overrides_enabled():
return _oauth_config_unavailable_response()
status = await get_oauth_config_status(session)
return JSONResponse({"credentials": status})
async def update_connector_oauth_config(
credential_key: str,
body: UpdateConnectorOAuthConfigBody,
user: User = Depends(require_permission("connectors:manage:access")),
session: AsyncSession = Depends(get_db_session),
):
"""Save (partial update) a workspace OAuth client id/secret override."""
from services.connector_oauth_config_service import get_oauth_config_status, set_oauth_config
if not is_workspace_oauth_overrides_enabled():
return _oauth_config_unavailable_response()
try:
await set_oauth_config(
session,
credential_key,
body.client_id,
body.client_secret,
user.db_user_id or user.user_id,
)
await session.commit()
except ValueError as e:
logger.error("[CONNECTOR] Invalid OAuth config update", error=str(e))
return JSONResponse({"error": "Unknown connector credential key"}, status_code=400)
return JSONResponse({"credentials": await get_oauth_config_status(session)})
async def delete_connector_oauth_config(
credential_key: str,
user: User = Depends(require_permission("connectors:manage:access")),
session: AsyncSession = Depends(get_db_session),
):
"""Clear a workspace OAuth client id/secret override, reverting to the env var."""
from services.connector_oauth_config_service import clear_oauth_config, get_oauth_config_status
if not is_workspace_oauth_overrides_enabled():
return _oauth_config_unavailable_response()
try:
await clear_oauth_config(session, credential_key, user.db_user_id or user.user_id)
await session.commit()
except ValueError as e:
logger.error("[CONNECTOR] Invalid OAuth config clear", error=str(e))
return JSONResponse({"error": "Unknown connector credential key"}, status_code=400)
return JSONResponse({"credentials": await get_oauth_config_status(session)})
async def connector_sync(
connector_type: str,
body: ConnectorSyncBody,
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
rbac=Depends(get_rbac_service),
):
"""Sync files from all active connections of a connector type"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
max_files = body.max_files
selected_files_raw = body.selected_files
selected_files = None
file_infos = None
if selected_files_raw:
if isinstance(selected_files_raw[0], str):
# Legacy format: just IDs
selected_files = selected_files_raw
else:
# New format: file objects with metadata
selected_files = [f.get("id") for f in selected_files_raw if f.get("id")]
file_infos = selected_files_raw
# Preview mode is opt-in from the connector upload UI (OSS only for now;
# SaaS deferred pending product approval). It applies to user-initiated
# ingests (explicit file selection), not automated re-sync.
preview_mode = body.preview and is_ingest_preview_enabled()
try:
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
)
logger.debug(
"Starting connector sync",
connector_type=connector_type,
max_files=max_files,
)
jwt_token = user.jwt_token
if body.shared and connector_type != "ibm_cos":
return JSONResponse(
{"error": "shared flag is only supported for the ibm_cos connector"},
status_code=400,
)
# Resolved for every sync, not just an explicitly shared one: orphan
# cleanup on a plain re-sync can also reach ownerless chunks (see
# delete_orphan_documents), and those are visible to every authenticated
# user under DLS regardless of who ingested them.
allow_anonymous_delete = await has_effective_permission(
request, user, rbac, "knowledge:delete:anonymous"
)
if body.shared and not allow_anonymous_delete:
return JSONResponse(
{"error": "Shared sync requires the knowledge:delete:anonymous permission"},
status_code=403,
)
# Get all active connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
)
active_connections = [conn for conn in connections if conn.is_active]
if not active_connections:
return JSONResponse(
{"error": f"No active {connector_type} connections found"},
status_code=404,
)
# Try the requested connection first, then the rest. Ordering rather than
# selecting outright keeps today's behaviour when that connection cannot
# authenticate: the sync falls through to another, or reports that none
# work, instead of failing later inside the connector.
candidate_connections = active_connections
if body.connection_id:
requested = [c for c in active_connections if c.connection_id == body.connection_id]
if requested:
candidate_connections = requested + [
c for c in active_connections if c.connection_id != body.connection_id
]
else:
logger.warning(
"Requested connection is not active for this connector type",
connector_type=connector_type,
connection_id=body.connection_id,
)
# Find the first connection that actually works
working_connection = None
for connection in candidate_connections:
logger.debug(
"Testing connection authentication",
connection_id=connection.connection_id,
)
try:
# Get the connector instance and test authentication
connector = await connector_service.get_connector(connection.connection_id)
if connector and await connector.authenticate():
working_connection = connection
logger.debug(
"Found working connection",
connection_id=connection.connection_id,
)
break
else:
logger.debug(
"Connection authentication failed",
connection_id=connection.connection_id,
)
except Exception as e:
logger.debug(
"Connection validation failed",
connection_id=connection.connection_id,
error=str(e),
)
continue
if not working_connection:
return JSONResponse(
{"error": f"No working {connector_type} connections found"},
status_code=404,
)
# Use the working connection
logger.debug(
"Starting sync with working connection",
connection_id=working_connection.connection_id,
)
# Branches set either ``task_id`` (single batch) or ``task_ids`` (the
# bucket_filter path may emit two batches: new files + changed files).
task_ids: list[str] = []
if selected_files:
# Explicit files selected (e.g., from file picker) - sync those specific files
from .documents import _ensure_index_exists
if not body.replace_duplicates and file_infos:
duplicate_check = await _classify_connector_duplicates(
connector=await connector_service.get_connector(
working_connection.connection_id
),
connector_type=connector_type,
selected_files_raw=file_infos,
session_manager=session_manager,
user_id=user.user_id,
jwt_token=jwt_token,
)
if duplicate_check["duplicate_count"] > 0:
file_infos = duplicate_check["non_duplicate_files"]
selected_files = [f["id"] for f in file_infos if f.get("id")]
if not selected_files:
return JSONResponse(
{
"status": "no_files",
"message": (
f"All {duplicate_check['duplicate_count']} selected file(s) "
"already exist. Nothing was synced."
),
"duplicate_names": duplicate_check["duplicate_names"],
"duplicate_count": duplicate_check["duplicate_count"],
"total_files": duplicate_check["total_files"],
},
status_code=200,
)
await _ensure_index_exists(jwt_token)
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
selected_files,
jwt_token=jwt_token,
file_infos=file_infos,
ingest_settings=body.settings,
replace_duplicates=body.replace_duplicates,
preview_mode=preview_mode,
shared=body.shared,
allow_anonymous_delete=allow_anonymous_delete,
)
elif body.sync_all or body.bucket_filter:
# Full ingest: discover and ingest all files (or files from specific buckets).
# Used by direct-sync providers on initial ingest or per-bucket sync.
logger.info(
"Full connector ingest requested",
connector_type=connector_type,
bucket_filter=body.bucket_filter,
)
connector = await connector_service.get_connector(working_connection.connection_id)
if body.bucket_filter:
# List only files from the requested buckets, then reconcile against
# what's already ingested so we re-fetch only NEW and CHANGED blobs
# (skipping unchanged ones), rather than re-downloading the whole
# container every time. Per-file dedup in ConnectorFileProcessor is a
# backstop, but it runs after download — this pre-filter avoids the
# redundant fetch/reprocess and the misleading "all files" task view.
scoped_connector = _connector_scoped_to_buckets(connector, body.bucket_filter)
all_files: list[dict[str, Any]] = []
page_token = None
while True:
result = await scoped_connector.list_files(page_token=page_token)
all_files.extend(result.get("files", []))
page_token = result.get("next_page_token")
if not page_token:
break
if not all_files:
return JSONResponse(
{
"status": "no_files",
"message": "No files found in the selected buckets.",
},
status_code=200,
)
# Carry the listing's names into the task so its file rows read
# "report.pdf" rather than the raw "<bucket>::<key>" id, which is
# all the task has until each blob is fetched.
infos_by_id = {
f["id"]: _connector_file_response(f) for f in all_files if f.get("id")
}
if not infos_by_id:
# Nothing addressable: every listed blob came back without an
# id, so there is no sync to start.
return JSONResponse(
{
"status": "no_files",
"message": "No files found in the selected buckets.",
},
status_code=200,
)
# Classify each remote blob as new / changed / unchanged.
existing_ids, _, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
existing_set = set(existing_ids)
state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
# "Overwrite duplicates", confirmed in the UI dialog, is about
# blobs whose name is already indexed but which this connector
# has never synced — a direct upload, another connector. Nothing
# we store about them can say whether the source is newer, so
# change detection cannot decide them and they are replaced
# outright. It deliberately does NOT cover blobs this connector
# already synced: those keep the modified-time/etag gate, so
# confirming an overwrite re-ingests the colliding files rather
# than every object in the container.
colliding_ids: set[str] = set()
if body.replace_duplicates:
# One winner per contested name. Two blobs in the same
# container can land on the same indexed filename (a bucket
# connector names a blob after its key's basename, so
# a/report.pdf and b/report.pdf are both "report.pdf"), and
# files within a task are ingested concurrently — replacing
# one document from two of them at once has no defined
# winner. The first in listing order takes the name; the
# rest fall through to the per-file backstop, which skips
# them as duplicates exactly as it does today.
claimed_names: set[str] = set()
collisions = await _bucket_blob_ids_with_indexed_filename(
all_files, session_manager, user.user_id, jwt_token
)
# "Synced here already?" is asked of these ids directly
# rather than read off existing_set, whose aggregation is
# capped: above the cap an ingested blob is missing from it,
# and would look like a collision with its own document.
synced_collisions = await _bucket_blob_ids_already_synced(
list(collisions), connector_type, session_manager, user.user_id, jwt_token
)
for fid, indexed_name in collisions.items():
if fid in synced_collisions or indexed_name in claimed_names:
continue
claimed_names.add(indexed_name)
colliding_ids.add(fid)
new_ids: list[str] = []
replace_ids: list[str] = []
for f in all_files:
fid = f.get("id")
if not fid:
continue
if fid in colliding_ids:
replace_ids.append(fid)
continue
status = classify_remote_file_change(
fid,
f.get("modified_time"),
fid in existing_set,
state_map,
f.get("etag"),
)
if status == "new":
new_ids.append(fid)
elif status == "changed":
replace_ids.append(fid)
# "unchanged" → skip; already ingested and not newer at source.
logger.info(
"Reconciled bucket selection",
connector_type=connector_type,
total=len(all_files),
new=len(new_ids),
replaced=len(replace_ids),
overwritten_duplicates=len(colliding_ids),
skipped=len(all_files) - len(new_ids) - len(replace_ids),
)
if not new_ids and not replace_ids:
return JSONResponse(
{
"status": "no_files",
"message": "All files in the selected buckets are already up to date.",
},
status_code=200,
)
# Two batches: new files are created; changed and overwritten
# files replace the indexed copy (replace_duplicates=True bypasses
# the filename-skip and deletes stale chunks before re-ingest).
# replace is batch-level, hence the split.
if new_ids:
task_ids.append(
await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
new_ids,
jwt_token=jwt_token,
file_infos=[infos_by_id[fid] for fid in new_ids],
ingest_settings=body.settings,
preview_mode=preview_mode,
shared=body.shared,
allow_anonymous_delete=allow_anonymous_delete,
)
)
if replace_ids:
task_ids.append(
await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
replace_ids,
jwt_token=jwt_token,
file_infos=[infos_by_id[fid] for fid in replace_ids],
ingest_settings=body.settings,
replace_duplicates=True,
preview_mode=preview_mode,
shared=body.shared,
allow_anonymous_delete=allow_anonymous_delete,
)
)
else:
# sync_all: ingest everything the connector can see
task_id = await connector_service.sync_connector_files(
working_connection.connection_id,
user.user_id,
max_files=max_files,
jwt_token=jwt_token,
ingest_settings=body.settings,
shared=body.shared,
allow_anonymous_delete=allow_anonymous_delete,
)
else:
# No files specified - sync only files already in OpenSearch for this connector
# This ensures deleted files stay deleted
(
existing_file_ids,
existing_filenames,
id_field,
) = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
if not existing_file_ids and not existing_filenames:
return JSONResponse(
{
"status": "no_files",
"message": f"No {connector_type} files to sync. Add files from the connector first.",
},
status_code=200,
)
sync_result = await _sync_existing_connector_files(
connector_type=connector_type,
working_connection=working_connection,
user_id=user.user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
existing_filenames=existing_filenames,
id_field=id_field,
ingest_settings=body.settings,
shared=body.shared,
allow_anonymous_delete=allow_anonymous_delete,
# Strict gating: skip orphan reconcile when sync is capped — we'd
# see a partial remote listing and delete legitimate files.
reconcile=body.max_files is None,
max_files=body.max_files,
)
if sync_result["outcome"] == "deleted_only":
return JSONResponse(
{
"status": "no_files",
"message": f"Deleted stale {connector_type} files; no remaining files to sync.",
},
status_code=200,
)
if sync_result["outcome"] == "up_to_date":
return JSONResponse(
{
"status": "no_files",
# Scoped to indexed files. Unlike the bucket_filter branch
# above, this path never enumerates new remote files, so
# it cannot claim the whole connector is up to date.
"message": (f"All indexed {connector_type} files are up to date."),
},
status_code=200,
)
task_id = sync_result["task_id"]
# The bucket_filter path may have already populated task_ids (new + changed
# batches); every other branch sets a single task_id.
if not task_ids:
task_ids = [task_id]
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE
)
return JSONResponse(
{
"task_ids": task_ids,
"status": "sync_started",
"message": f"Started syncing files from 1 {connector_type} connection",
"connections_synced": len(task_ids),
},
status_code=201,
)
except Exception as e:
logger.error("Connector sync failed", error=str(e))
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED
)
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
async def connector_status(
connector_type: str,
request: Request,
connector_service=Depends(get_connector_service),
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
):
"""Get connector status for authenticated user"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
# Get connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
)
# Get the connector for each connection and verify authentication
connection_details = {}
verified_active_connections = []
for connection in connections:
try:
connector = await connector_service._get_connector(connection.connection_id)
if connector is not None:
# Actually verify the connection by trying to authenticate
is_authenticated = await connector.authenticate()
# Get base URL if available (for SharePoint/OneDrive connectors)
base_url = None
if hasattr(connector, "base_url"):
base_url = connector.base_url
logger.debug(
f"connector_status: Got base_url from connector.base_url: {base_url}"
)
elif hasattr(connector, "sharepoint_url"):
base_url = connector.sharepoint_url # Backward compatibility
logger.debug(
f"connector_status: Got base_url from connector.sharepoint_url: {base_url}"
)
else:
logger.debug(
"connector_status: Connector has no base_url or sharepoint_url attribute"
)
connection_details[connection.connection_id] = {
"client_id": connector.get_client_id(),
"is_authenticated": is_authenticated,
"base_url": base_url,
}
if is_authenticated and connection.is_active:
verified_active_connections.append(connection)
else:
connection_details[connection.connection_id] = {
"client_id": None,
"is_authenticated": False,
"base_url": None,
}
except Exception as e:
logger.warning(
"Could not verify connector authentication",
connection_id=connection.connection_id,
error=str(e),
)
connection_details[connection.connection_id] = {
"client_id": None,
"is_authenticated": False,
"base_url": None,
}
# Only count connections that are both active AND actually authenticated
has_authenticated_connection = len(verified_active_connections) > 0
# Check if OAuth credentials are configured in environment (for OAuth connectors only)
has_env_credentials = connector_service.connection_manager.has_env_credentials(connector_type)
return JSONResponse(
{
"connector_type": connector_type,
"authenticated": has_authenticated_connection,
"status": "connected" if has_authenticated_connection else "not_connected",
"has_env_credentials": has_env_credentials,
"connections": [
{
"connection_id": conn.connection_id,
"name": conn.name,
"client_id": connection_details.get(conn.connection_id, {}).get("client_id"),
"is_active": conn.is_active
and connection_details.get(conn.connection_id, {}).get(
"is_authenticated", False
),
"is_authenticated": connection_details.get(conn.connection_id, {}).get(
"is_authenticated", False
),
"base_url": connection_details.get(conn.connection_id, {}).get("base_url"),
"created_at": conn.created_at.isoformat(),
"last_sync": conn.last_sync.isoformat() if conn.last_sync else None,
}
for conn in connections
],
}
)
# Drive watches registered with a legacy webhook URL may point at
# /connectors/google/webhook; accept them until those channels expire.
LEGACY_WEBHOOK_TYPE_ALIASES = {"google": "google_drive"}
async def connector_webhook(
connector_type: str,
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
session: AsyncSession = Depends(get_db_session),
):
"""Handle webhook notifications from any connector type"""
canonical_type = LEGACY_WEBHOOK_TYPE_ALIASES.get(connector_type)
if canonical_type:
logger.warning(
"Legacy webhook connector type received, aliasing",
received=connector_type,
canonical=canonical_type,
)
connector_type = canonical_type
if denied := await _connector_access_denied(request, session, connector_type):
return denied
# Handle webhook validation (connector-specific)
temp_config = {"token_file": "temp.json"}
from connectors.connection_manager import ConnectionConfig
temp_connection = ConnectionConfig(
connection_id="temp",
connector_type=str(connector_type),
name="temp",
config=temp_config,
)
try:
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_RECV
)
temp_connector = connector_service.connection_manager._create_connector(temp_connection)
validation_response = temp_connector.handle_webhook_validation(
request.method, dict(request.headers), dict(request.query_params)
)
if validation_response:
return PlainTextResponse(validation_response)
except (NotImplementedError, ValueError):
# Connector type not found or validation not needed
pass
try:
# Get the raw payload and headers
payload = {}
headers = dict(request.headers)
if request.method == "POST":
content_type = headers.get("content-type", "").lower()
if "application/json" in content_type:
payload = await request.json()
else:
# Some webhooks send form data or plain text
body = await request.body()
payload = {"raw_body": body.decode("utf-8") if body else ""}
else:
# GET webhooks use query params
payload = dict(request.query_params)
# Add headers to payload for connector processing
payload["_headers"] = headers
payload["_method"] = request.method
logger.info("Webhook notification received", connector_type=connector_type)
# Extract channel/subscription ID using connector-specific method
try:
temp_connector = connector_service.connection_manager._create_connector(temp_connection)
channel_id = temp_connector.extract_webhook_channel_id(payload, headers)
except (NotImplementedError, ValueError):
channel_id = None
if not channel_id:
logger.warning("No channel ID found in webhook", connector_type=connector_type)
return JSONResponse({"status": "ignored", "reason": "no_channel_id"})
# Find the specific connection for this webhook
connection = await connector_service.connection_manager.get_connection_by_webhook_id(
channel_id
)
if not connection or not connection.is_active:
logger.info("Unknown webhook channel, will auto-expire", channel_id=channel_id)
return JSONResponse({"status": "ignored_unknown_channel", "channel_id": channel_id})
# Process webhook for the specific connection
try:
# Get the connector instance
connector = await connector_service._get_connector(connection.connection_id)
if not connector:
logger.error(
"Could not get connector for connection",
connection_id=connection.connection_id,
)
return JSONResponse({"status": "error", "reason": "connector_not_found"})
# Let the connector handle the webhook and return affected file IDs
affected_files = await connector.handle_webhook(payload)
user = session_manager.get_user(connection.user_id)
jwt_token = user.jwt_token if user else None
# Scope guard: a connection's picker selection is not persisted, so
# the connector can't filter the change feed to it. Restrict webhook
# ingestion to files ALREADY indexed for this connector — the same
# durable scope the no-selection manual sync uses. This stops a stray
# change (even just opening a file) from auto-ingesting a file the
# user never selected. Deletions of indexed files still pass (they
# remain in the index until cleaned up) so chunk-cleanup runs.
in_scope: list[str] = []
if affected_files:
indexed_ids, _filenames, _id_field = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=connection.user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
indexed_set = set(indexed_ids)
in_scope = [f for f in affected_files if f in indexed_set]
if in_scope:
logger.info(
"Webhook connection files affected",
connection_id=connection.connection_id,
affected_count=len(affected_files),
in_scope_count=len(in_scope),
)
# Trigger incremental sync for affected files. The webhook fires
# because the file changed, so replace the indexed copy instead of
# tripping the duplicate-filename guard meant for manual uploads.
# shared=None like the other re-sync paths: no user is in the
# loop to express an intent, so each file keeps the sharing state
# it already has.
# allow_anonymous_delete=False for the same reason: nobody is
# here to authorize removing a document the whole instance can
# see, so a webhook replaces only what the connection's owner
# owns. A file indexed as shared is left as it is until someone
# holding the permission syncs it; no connector that delivers
# webhooks can index one today.
task_id = await connector_service.sync_specific_files(
connection.connection_id,
connection.user_id,
in_scope,
jwt_token=jwt_token,
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=None,
allow_anonymous_delete=False,
)
result = {
"connection_id": connection.connection_id,
"task_id": task_id,
"affected_files": len(in_scope),
}
elif affected_files:
# Changes detected, but none are within the indexed scope —
# ignore so unselected files are not auto-ingested.
logger.info(
"Webhook changes outside synced scope, ignored",
connection_id=connection.connection_id,
affected_count=len(affected_files),
in_scope_count=0,
)
result = {
"connection_id": connection.connection_id,
"action": "ignored",
"reason": "out_of_scope",
}
else:
# No specific files identified - just log the webhook
logger.info(
"Webhook general change detected, no specific files",
connection_id=connection.connection_id,
)
result = {
"connection_id": connection.connection_id,
"action": "logged_only",
"reason": "no_specific_files",
}
return JSONResponse(
{
"status": "processed",
"connector_type": connector_type,
"channel_id": channel_id,
**result,
}
)
except Exception as e:
logger.exception(
"[CONNECTOR] Failed to process webhook",
connection_id=connection.connection_id,
)
return JSONResponse(
{
"status": "error",
"connector_type": connector_type,
"channel_id": channel_id,
"error": str(e),
},
status_code=500,
)
except Exception as e:
logger.error("Webhook processing failed", error=str(e))
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_FAILED
)
return JSONResponse({"error": f"Webhook processing failed: {str(e)}"}, status_code=500)
async def connector_disconnect(
connector_type: str,
request: Request,
connector_service=Depends(get_connector_service),
user: User = Depends(require_permission("connectors:delete:own")),
session: AsyncSession = Depends(get_db_session),
):
"""Disconnect a connector by deleting its connection"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
try:
# Get connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
)
if not connections:
return JSONResponse(
{"error": f"No {connector_type} connections found"},
status_code=404,
)
# Delete all connections for this connector type and user
deleted_count = 0
for connection in connections:
try:
# Get the connector to cleanup any subscriptions
connector = await connector_service._get_connector(connection.connection_id)
if connector and hasattr(connector, "cleanup_subscription"):
subscription_id = connection.config.get("webhook_channel_id")
if subscription_id:
try:
await connector.cleanup_subscription(subscription_id)
except Exception as e:
logger.warning(
"Failed to cleanup subscription",
connection_id=connection.connection_id,
error=str(e),
)
except Exception as e:
logger.warning(
"Could not get connector for cleanup",
connection_id=connection.connection_id,
error=str(e),
)
# Delete the connection
success = await connector_service.connection_manager.delete_connection(
connection.connection_id
)
if success:
deleted_count += 1
logger.info(
"Disconnected connector",
connector_type=connector_type,
user_id=user.user_id,
deleted_count=deleted_count,
)
return JSONResponse(
{
"status": "disconnected",
"connector_type": connector_type,
"deleted_connections": deleted_count,
}
)
except Exception as e:
logger.error(
"Failed to disconnect connector",
connector_type=connector_type,
error=str(e),
)
return JSONResponse(
{"error": f"Disconnect failed: {str(e)}"},
status_code=500,
)
# ---------------------------------------------------------------------------
async def sync_all_connectors(
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
rbac=Depends(get_rbac_service),
):
"""
Sync files from all active cloud connector connections.
"""
try:
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
)
jwt_token = user.jwt_token
# Sync-all reconciles orphans across every connector type, so it can
# reach ownerless chunks too — same gate as the per-connector endpoint.
allow_anonymous_delete = await has_effective_permission(
request, user, rbac, "knowledge:delete:anonymous"
)
all_task_ids = []
synced_connectors = []
skipped_connectors = []
deleted_only_connectors = []
errors = []
connector_types = await _allowed_connector_types_for_request(
request, session, _cloud_connector_types()
)
for connector_type in connector_types:
try:
# First, get existing file IDs/filenames from OpenSearch for this connector type
(
existing_file_ids,
existing_filenames,
id_field,
) = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
if not existing_file_ids and not existing_filenames:
logger.debug(
"No existing files in OpenSearch for connector type, skipping",
connector_type=connector_type,
)
skipped_connectors.append(connector_type)
continue
# Get all active connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
)
active_connections = [conn for conn in connections if conn.is_active]
if not active_connections:
logger.debug(
"No active connections for connector type",
connector_type=connector_type,
)
continue
# Find the first connection that actually works
working_connection = None
for connection in active_connections:
try:
connector = await connector_service.get_connector(connection.connection_id)
if connector and await connector.authenticate():
working_connection = connection
break
except Exception as e:
logger.debug(
"Connection validation failed",
connection_id=connection.connection_id,
error=str(e),
)
continue
if not working_connection:
logger.debug(
"No working connection for connector type",
connector_type=connector_type,
)
continue
sync_result = await _sync_existing_connector_files(
connector_type=connector_type,
working_connection=working_connection,
user_id=user.user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
existing_filenames=existing_filenames,
id_field=id_field,
allow_anonymous_delete=allow_anonymous_delete,
)
if sync_result["outcome"] == "deleted_only":
deleted_only_connectors.append(connector_type)
continue
if sync_result["outcome"] == "up_to_date":
# Nothing changed at source — already up to date.
skipped_connectors.append(connector_type)
continue
task_id = sync_result["task_id"]
all_task_ids.append(task_id)
synced_connectors.append(connector_type)
logger.info(
"Started sync for connector type",
connector_type=connector_type,
task_id=task_id,
file_count=len(existing_file_ids)
if existing_file_ids
else len(existing_filenames),
)
except Exception as e:
logger.error(
"Failed to sync connector type",
connector_type=connector_type,
error=str(e),
)
errors.append({"connector_type": connector_type, "error": str(e)})
if not all_task_ids and not errors:
if deleted_only_connectors:
return JSONResponse(
{
"status": "no_files",
"message": "Deleted stale cloud files; no remaining files to sync.",
"skipped_connectors": skipped_connectors if skipped_connectors else None,
"deleted_only_connectors": deleted_only_connectors,
},
status_code=200,
)
if skipped_connectors:
return JSONResponse(
{
"status": "no_files",
"message": "No files to sync. Add files from cloud connectors first.",
"skipped_connectors": skipped_connectors,
},
status_code=200,
)
return JSONResponse(
{"error": "No active cloud connector connections found"},
status_code=404,
)
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE
)
return JSONResponse(
{
"task_ids": all_task_ids,
"status": "sync_started",
"message": f"Started syncing files from {len(synced_connectors)} cloud connector(s)",
"synced_connectors": synced_connectors,
"skipped_connectors": skipped_connectors if skipped_connectors else None,
"errors": errors if errors else None,
},
status_code=201,
)
except Exception as e:
logger.error("Sync all connectors failed", error=str(e))
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED
)
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
def _cloud_connector_types() -> list[str]:
from connectors.registry import get_connector_classes
return [cls.CONNECTOR_TYPE for cls in get_connector_classes()]
class ConnectorSyncPreview(NamedTuple):
"""What a sync would do to one connector type, without doing any of it.
``orphans`` is None when strict gating aborted the pass (so the caller can
say "couldn't determine" rather than "nothing to delete"). ``updates`` is
None when this connector can't answer the question ahead of time — see
``_preview_for_connector_type``.
"""
orphans: list[dict[str, str]] | None
updates: list[dict[str, str]] | None
synced_count: int
async def _preview_for_connector_type(
connector_type: str,
user_id: str,
connector_service,
session_manager,
jwt_token: str | None,
) -> ConnectorSyncPreview:
"""Compute what a sync would delete and re-ingest, without touching anything.
Deletions and updates come out of a single remote listing, so previewing
costs the same as the orphan pass used to.
Updates are only knowable for connectors that declare timestamp change
detection: they compare a listing against stored signals. ``replace_always``
connectors decide per file after downloading it, so their update count is
reported as unavailable rather than guessed at.
"""
existing_file_ids, existing_filenames, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
synced_count = len(existing_file_ids) if existing_file_ids else len(existing_filenames)
if not existing_file_ids:
# No document_ids to diff against (e.g. Langflow-only ingest). Filename-only
# fallback can't detect orphans safely — surface empty list.
return ConnectorSyncPreview(orphans=[], updates=None, synced_count=synced_count)
id_to_filename = await get_synced_id_to_filename_map(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
remote_files = await list_remote_files_for_connector_type(
connector_type=connector_type,
user_id=user_id,
connector_service=connector_service,
existing_file_ids=existing_file_ids,
)
orphans = await compute_orphans_for_connector_type(
connector_type=connector_type,
user_id=user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
id_to_filename=id_to_filename,
remote_files=remote_files,
)
if orphans is not None:
synced_count = max(0, synced_count - len(orphans))
updates = None
if remote_files is not None and _connector_uses_timestamp_change_detection(connector_type):
state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
)
updates = [
{"document_id": fid, "filename": id_to_filename.get(fid, "")}
for fid in existing_file_ids
if fid in remote_files
and classify_remote_file_change(
fid,
remote_files[fid].get("modified_time"),
True,
state_map,
remote_files[fid].get("etag"),
)
== "changed"
]
return ConnectorSyncPreview(orphans=orphans, updates=updates, synced_count=synced_count)
async def connector_sync_preview(
connector_type: str,
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
):
"""Preview the impact of syncing a connector type without performing any
deletion or ingest. Returns the orphan files (present in OpenSearch but no
longer at the source) and the files whose source copy has changed, both by
filename, plus the total synced count.
"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
try:
preview = await _preview_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=user.jwt_token,
)
return JSONResponse(
{
"connector_type": connector_type,
"synced_count": preview.synced_count,
"orphans": preview.orphans or [],
"orphans_available": preview.orphans is not None,
"updates": preview.updates or [],
# False means this connector can't tell ahead of time (it decides
# per file during ingest), not that nothing will be updated.
"updates_available": preview.updates is not None,
},
status_code=200,
)
except Exception as e:
logger.error("Sync preview failed", connector_type=connector_type, error=str(e))
return JSONResponse({"error": f"Sync preview failed: {str(e)}"}, status_code=500)
async def connectors_sync_all_preview(
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("connectors:use")),
session: AsyncSession = Depends(get_db_session),
):
"""Preview the impact of sync-all-connectors across every cloud connector
type. Returns orphan and changed filenames grouped by connector_type plus a
per-type synced count.
"""
try:
orphans_by_type: dict[str, list[dict[str, str]]] = {}
synced_count_by_type: dict[str, int] = {}
orphans_available_by_type: dict[str, bool] = {}
updates_by_type: dict[str, list[dict[str, str]]] = {}
updates_available_by_type: dict[str, bool] = {}
connector_types = await _allowed_connector_types_for_request(
request, session, _cloud_connector_types()
)
for connector_type in connector_types:
try:
preview = await _preview_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
session_manager=session_manager,
jwt_token=user.jwt_token,
)
except Exception as e:
logger.warning(
"Sync-all preview: per-connector failure",
connector_type=connector_type,
error=str(e),
)
preview = ConnectorSyncPreview(orphans=None, updates=None, synced_count=0)
# Only include connector types that have something synced.
if preview.synced_count == 0 and not preview.orphans:
continue
synced_count_by_type[connector_type] = preview.synced_count
orphans_by_type[connector_type] = preview.orphans or []
orphans_available_by_type[connector_type] = preview.orphans is not None
updates_by_type[connector_type] = preview.updates or []
updates_available_by_type[connector_type] = preview.updates is not None
return JSONResponse(
{
"orphans_by_type": orphans_by_type,
"synced_count_by_type": synced_count_by_type,
"orphans_available_by_type": orphans_available_by_type,
"updates_by_type": updates_by_type,
"updates_available_by_type": updates_available_by_type,
},
status_code=200,
)
except Exception as e:
logger.error("Sync-all preview failed", error=str(e))
return JSONResponse({"error": f"Sync-all preview failed: {str(e)}"}, status_code=500)
async def connector_token(
connector_type: str,
connection_id: str,
request: Request,
connector_service=Depends(get_connector_service),
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
):
"""Get access token for connector API calls (e.g., Pickers)."""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
url_connector_type = connector_type
try:
# 1) Load the connection and verify ownership
connection = await connector_service.connection_manager.get_connection(connection_id)
if not connection or connection.user_id != user.user_id:
return JSONResponse({"error": "Connection not found"}, status_code=404)
# 2) Get the ACTUAL connector instance/type for this connection_id
connector = await connector_service._get_connector(connection_id)
if not connector:
return JSONResponse(
{
"error": f"Connector not available - authentication may have failed for {url_connector_type}"
},
status_code=404,
)
real_type = getattr(connector, "type", None) or getattr(connection, "connector_type", None)
if real_type is None:
return JSONResponse({"error": "Unable to determine connector type"}, status_code=500)
# Optional: warn if URL path type disagrees with real type
if url_connector_type and url_connector_type != real_type:
# You can downgrade this to debug if you expect cross-routing.
return JSONResponse(
{
"error": "Connector type mismatch",
"detail": {
"requested_type": url_connector_type,
"actual_type": real_type,
"hint": "Call the token endpoint using the correct connector_type for this connection_id.",
},
},
status_code=400,
)
# 3) Branch by the actual connector type
# GOOGLE DRIVE (google-auth)
if real_type == "google_drive" and hasattr(connector, "oauth"):
await connector.oauth.load_credentials()
if connector.oauth.creds and connector.oauth.creds.valid:
expires_in = None
try:
if connector.oauth.creds.expiry:
import time
expires_in = max(
0, int(connector.oauth.creds.expiry.timestamp() - time.time())
)
except Exception:
expires_in = None
return JSONResponse(
{
"access_token": connector.oauth.creds.token,
"expires_in": expires_in,
}
)
return JSONResponse({"error": "Invalid or expired credentials"}, status_code=401)
# ONEDRIVE / SHAREPOINT (MSAL or custom)
if real_type in ("onedrive", "sharepoint") and hasattr(connector, "oauth"):
# Ensure cache/credentials are loaded before trying to use them
try:
# Prefer a dedicated is_authenticated() that loads cache internally
if hasattr(connector.oauth, "is_authenticated"):
ok = await connector.oauth.is_authenticated()
else:
# Fallback: try to load credentials explicitly if available
ok = True
if hasattr(connector.oauth, "load_credentials"):
ok = await connector.oauth.load_credentials()
if not ok:
return JSONResponse({"error": "Not authenticated"}, status_code=401)
# Check if a specific resource is requested (for SharePoint File Picker v8)
# The File Picker requires a token with SharePoint as the audience, not Graph
resource = request.query_params.get("resource")
if resource and is_valid_sharepoint_url(resource):
# SharePoint File Picker v8 needs a SharePoint-scoped token
logger.info(f"Acquiring SharePoint-scoped token for resource: {resource}")
if hasattr(connector.oauth, "get_access_token_for_resource"):
access_token = connector.oauth.get_access_token_for_resource(resource)
else:
# Fallback for connectors without resource-specific token support
access_token = connector.oauth.get_access_token()
else:
# Default: Microsoft Graph token
access_token = connector.oauth.get_access_token()
# MSAL result has expiry, but we’re returning a raw token; keep expires_in None for simplicity
return JSONResponse({"access_token": access_token, "expires_in": None})
except ValueError as e:
# Typical when acquire_token_silent fails (e.g., needs re-auth)
return JSONResponse(
{"error": f"Failed to get access token: {str(e)}"}, status_code=401
)
except Exception as e:
return JSONResponse({"error": f"Authentication error: {str(e)}"}, status_code=500)
return JSONResponse(
{"error": "Token not available for this connector type"}, status_code=400
)
except Exception as e:
logger.error("Error getting connector token", exc_info=True)
return JSONResponse({"error": str(e)}, status_code=500)
async def browse_connection_files(
connector_type: str,
connection_id: str,
request: Request,
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(get_current_user),
session: AsyncSession = Depends(get_db_session),
bucket: str | None = None,
search: str | None = None,
page_token: str | None = None,
max_files: int = 100,
):
"""
Browse remote files in a connector with ingestion status.
Lists files from the remote source (e.g., S3 bucket) and marks each
as ingested or not by cross-referencing with OpenSearch.
"""
if denied := await _connector_access_denied(request, session, connector_type):
return denied
try:
connector = await connector_service.get_connector(connection_id)
if not connector:
return JSONResponse(
{"error": "Connection not found or connector unavailable"},
status_code=404,
)
if not await connector.authenticate():
return JSONResponse(
{"error": "Connector authentication failed"},
status_code=401,
)
# Scope the listing to the requested bucket without mutating the
# shared cached connector instance.
listing_connector = connector
if bucket and hasattr(connector, "bucket_names"):
listing_connector = _connector_scoped_to_buckets(connector, [bucket])
files_result = await listing_connector.list_files(
page_token=page_token, max_files=max_files
)
remote_files = files_result.get("files", [])
next_page_token = files_result.get("next_page_token")
# Filter by filename search if provided
if search:
search_lower = search.lower()
remote_files = [f for f in remote_files if search_lower in f.get("name", "").lower()]
# Get already-ingested file IDs from OpenSearch
ingested_ids, ingested_filenames, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=user.jwt_token,
)
ingested_set = set(ingested_ids) | set(ingested_filenames)
# Stored change-detection state per ingested source id, for "update
# available" detection.
state_map = await get_synced_file_state_map(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
jwt_token=user.jwt_token,
)
# Merge ingestion status into remote file list
enriched_files = []
for f in remote_files:
file_id = f.get("id", "")
is_ingested = file_id in ingested_set or f.get("name", "") in ingested_set
# "Update available": ingested, but the source copy differs from what we
# indexed. Uses the same classification as sync so the browser and the
# sync agree. The frontend keeps unchanged files disabled but lets the
# user re-ingest stale ones (with replace_duplicates).
is_stale = (
classify_remote_file_change(
file_id, f.get("modified_time"), is_ingested, state_map, f.get("etag")
)
== "changed"
)
enriched_files.append(
{
"id": file_id,
"name": f.get("name", ""),
"bucket": f.get("bucket", ""),
"key": f.get("key", ""),
"size": f.get("size", 0),
"modified_time": f.get("modified_time", ""),
"is_ingested": is_ingested,
"is_stale": is_stale,
}
)
return JSONResponse(
{
"files": enriched_files,
"next_page_token": next_page_token,
"total_remote": len(enriched_files),
"total_ingested": sum(1 for f in enriched_files if f["is_ingested"]),
}
)
except Exception as e:
logger.error(
"Failed to browse connection files",
connector_type=connector_type,
connection_id=connection_id,
error=str(e),
)
return JSONResponse(
{"error": f"Failed to browse files: {str(e)}"},
status_code=500,
)