OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Skip deleted connector files; return orphan IDs (#1592)
Edwin Josed2550408ee7b
diff --git a/.gitignore b/.gitignoreindex 6f3ded523..ebc5100db 100644--- a/.gitignore+++ b/.gitignore@@ -24,7 +24,6 @@ wheels/!src/tui/_assets/flows/*.json!src/tui/_assets/flows/components/*.json!frontend/*.json-!**/.claude-plugin/*.json.DS_Store/config/@@ -39,20 +38,16 @@ langflow-data/node_modules# AI Tools-# Per-contributor Claude Code state is ignored, but shared skills and-# plugin manifests (.claude/skills, .claude-plugin) are tracked.-/.claude/*-!/.claude/skills/+/.claude+/CLAUDE.md/.gemini/GEMINI.md/.bob+/AGENTS.mddocuments/warmup_ocr.pdfdocuments/openrag-documentation.pdfdocuments/ibm_anthropic.pdfdocuments/docling.pdf/opensearch-data-new-lf/opensearch-data2-data/openrag.db-data/openrag.db*.db-/datadiff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsindex fe2c7ef2e..e40508211 100644--- a/frontend/app/api/mutations/useSyncConnector.ts+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -13,6 +13,27 @@ interface SyncResponse {errors?: Array<{ connector_type: string; error: string }> | null;}+export interface OrphanFile {+ document_id: string;+ filename: string;+}++export interface SyncPreviewResponse {+ connector_type: string;+ synced_count: number;+ orphans: OrphanFile[];+ /** False when strict gating aborted orphan detection (e.g. an active+ * connection was unauthenticated). UI should reflect that deletions+ * cannot be predicted in that case. */+ orphans_available: boolean;+}++export interface SyncAllPreviewResponse {+ orphans_by_type: Record<string, OrphanFile[]>;+ synced_count_by_type: Record<string, number>;+ orphans_available_by_type: Record<string, boolean>;+}+// Sync all cloud connectorsconst syncAllConnectors = async (): Promise<SyncResponse> => {const response = await fetch("/api/connectors/sync-all", {@@ -92,3 +113,44 @@ export const useSyncConnector = () => {},});};++const syncConnectorPreview = async (+ connectorType: string,+): Promise<SyncPreviewResponse> => {+ const response = await fetch(+ `/api/connectors/${connectorType}/sync-preview`,+ {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ },+ );++ if (!response.ok) {+ const error = await response.json().catch(() => ({}));+ throw new Error(+ error.error || `Failed to preview sync for ${connectorType}`,+ );+ }++ return response.json();+};++const syncAllConnectorsPreview = async (): Promise<SyncAllPreviewResponse> => {+ const response = await fetch("/api/connectors/sync-all-preview", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ });++ if (!response.ok) {+ const error = await response.json().catch(() => ({}));+ throw new Error(error.error || "Failed to preview sync");+ }++ return response.json();+};++export const useSyncConnectorPreview = () =>+ useMutation({ mutationFn: syncConnectorPreview });++export const useSyncAllConnectorsPreview = () =>+ useMutation({ mutationFn: syncAllConnectorsPreview });diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex a5f0a4efc..ea5f9af27 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -55,9 +55,14 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";import IBMCOSIcon from "../../components/icons/ibm-cos-icon";import OneDriveIcon from "../../components/icons/one-drive-logo";import SharePointIcon from "../../components/icons/share-point-logo";+import { SyncConfirmDialog } from "../../components/sync-confirm-dialog";import { useDeleteDocument } from "../api/mutations/useDeleteDocument";import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";-import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";+import {+ type SyncAllPreviewResponse,+ useSyncAllConnectors,+ useSyncAllConnectorsPreview,+} from "../api/mutations/useSyncConnector";/** List-files uses term filters; "*" means "any" in the UI — do not send it literally. */function listFilesFilterParam(values?: string[]): string | undefined {@@ -119,7 +124,51 @@ function SearchPage() {const deleteDocumentMutation = useDeleteDocument();const syncAllConnectorsMutation = useSyncAllConnectors();+ const syncAllPreviewMutation = useSyncAllConnectorsPreview();const refreshOpenragDocsMutation = useRefreshOpenragDocs();+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);+ const [syncPreview, setSyncPreview] = useState<SyncAllPreviewResponse | null>(+ null,+ );++ const handleOpenSyncDialog = useCallback(async () => {+ setSyncPreview(null);+ setSyncDialogOpen(true);+ try {+ const preview = await syncAllPreviewMutation.mutateAsync();+ setSyncPreview(preview);+ } catch (error) {+ setSyncDialogOpen(false);+ toast.error(+ error instanceof Error ? error.message : "Failed to preview sync",+ );+ }+ }, [syncAllPreviewMutation]);++ const handleConfirmSync = useCallback(async () => {+ try {+ const result = await syncAllConnectorsMutation.mutateAsync();+ if (result.status === "no_files") {+ toast.info(+ result.message ||+ "No cloud files to sync. Add files from cloud connectors first.",+ );+ } else if (+ result.synced_connectors &&+ result.synced_connectors.length > 0+ ) {+ toast.success(+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,+ );+ } else if (result.errors && result.errors.length > 0) {+ toast.error("Some connectors failed to sync");+ }+ } catch (error) {+ toast.error(+ error instanceof Error ? error.message : "Failed to sync connectors",+ );+ }+ }, [syncAllConnectorsMutation]);useEffect(() => {refreshTasks();@@ -761,36 +810,14 @@ function SearchPage() {type="button"variant="outline"className="rounded-lg flex-shrink-0"- disabled={syncAllConnectorsMutation.isPending}- onClick={async () => {- try {- toast.info("Syncing all cloud connectors...");- const result = await syncAllConnectorsMutation.mutateAsync();- if (result.status === "no_files") {- toast.info(- result.message ||- "No cloud files to sync. Add files from cloud connectors first.",- );- } else if (- result.synced_connectors &&- result.synced_connectors.length > 0- ) {- toast.success(- `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,- );- } else if (result.errors && result.errors.length > 0) {- toast.error("Some connectors failed to sync");- }- } catch (error) {- toast.error(- error instanceof Error- ? error.message- : "Failed to sync connectors",- );- }- }}+ disabled={+ syncAllConnectorsMutation.isPending ||+ syncAllPreviewMutation.isPending+ }+ onClick={handleOpenSyncDialog}>- {syncAllConnectorsMutation.isPending ? (+ {syncAllConnectorsMutation.isPending ||+ syncAllPreviewMutation.isPending ? (<><RefreshCw className="h-4 w-4 mr-2 animate-spin" />Syncing...@@ -961,6 +988,18 @@ function SearchPage() {<p className="my-2">Documents to be deleted:</p>{formatFilesToDelete(selectedRows)}</DeleteConfirmationDialog>++ <SyncConfirmDialog+ open={syncDialogOpen}+ onOpenChange={setSyncDialogOpen}+ onConfirm={handleConfirmSync}+ isLoading={syncAllPreviewMutation.isPending || syncPreview === null}+ isSyncing={syncAllConnectorsMutation.isPending}+ isSyncAll+ orphansByType={syncPreview?.orphans_by_type}+ orphansAvailableByType={syncPreview?.orphans_available_by_type}+ syncedCountByType={syncPreview?.synced_count_by_type}+ /></>);}diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsxindex 62322d89e..a7a897c5c 100644--- a/frontend/components/knowledge-actions-dropdown.tsx+++ b/frontend/components/knowledge-actions-dropdown.tsx@@ -5,7 +5,11 @@ import { useRouter } from "next/navigation";import { useMemo, useState } from "react";import { toast } from "sonner";import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";-import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";+import {+ type SyncPreviewResponse,+ useSyncConnector,+ useSyncConnectorPreview,+} from "@/app/api/mutations/useSyncConnector";import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";import {DropdownMenu,@@ -24,6 +28,7 @@ import { usePermissions } from "@/hooks/use-permissions";import { formatFilesToDelete } from "@/lib/format-files-to-delete";import { DeleteConfirmationDialog } from "./delete-confirmation-dialog";import { RequirePermission } from "./require-permission";+import { SyncConfirmDialog } from "./sync-confirm-dialog";import { Button } from "./ui/button";interface KnowledgeActionsDropdownProps {@@ -46,6 +51,11 @@ export const KnowledgeActionsDropdown = ({const [showDeleteDialog, setShowDeleteDialog] = useState(false);const deleteDocumentMutation = useDeleteDocument();const syncConnectorMutation = useSyncConnector();+ const syncPreviewMutation = useSyncConnectorPreview();+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);+ const [syncPreview, setSyncPreview] = useState<SyncPreviewResponse | null>(+ null,+ );const { data: connectors = [] } = useGetConnectorsQuery();const router = useRouter();@@ -81,11 +91,24 @@ export const KnowledgeActionsDropdown = ({}};- const handleSync = async () => {+ const handleOpenSyncDialog = async () => {if (!connectorType || !isConnected) return;+ setSyncPreview(null);+ setSyncDialogOpen(true);+ try {+ const preview = await syncPreviewMutation.mutateAsync(connectorType);+ setSyncPreview(preview);+ } catch (error) {+ setSyncDialogOpen(false);+ toast.error(+ error instanceof Error ? error.message : "Failed to preview sync",+ );+ }+ };+ const handleConfirmSync = async () => {+ if (!connectorType) return;try {- toast.info(`Syncing ${connectorType}...`);const result = await syncConnectorMutation.mutateAsync({ connectorType });if (result.status === "no_files") {toast.info(result.message || `No ${connectorType} files to sync.`);@@ -129,16 +152,21 @@ export const KnowledgeActionsDropdown = ({<div className="w-full"><DropdownMenuItemclassName="text-primary focus:text-primary cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed"- disabled={syncConnectorMutation.isPending || !isConnected}+ disabled={+ syncConnectorMutation.isPending ||+ syncPreviewMutation.isPending ||+ !isConnected+ }onClick={(e) => {if (!isConnected) {e.preventDefault();return;}- handleSync();+ handleOpenSyncDialog();}}>- {syncConnectorMutation.isPending ? (+ {syncConnectorMutation.isPending ||+ syncPreviewMutation.isPending ? (<><RefreshCw className="h-4 w-4 mr-2 animate-spin" />Syncing...@@ -196,6 +224,17 @@ export const KnowledgeActionsDropdown = ({<p className="my-2">Document to be deleted:</p>{formatFilesToDelete([{ filename }])}</DeleteConfirmationDialog>++ <SyncConfirmDialog+ open={syncDialogOpen}+ onOpenChange={setSyncDialogOpen}+ onConfirm={handleConfirmSync}+ isLoading={syncPreviewMutation.isPending || syncPreview === null}+ isSyncing={syncConnectorMutation.isPending}+ connectorType={connectorType}+ orphans={syncPreview?.orphans}+ syncedCount={syncPreview?.synced_count}+ /></>);};diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsxindex 03a449173..d7f7b3959 100644--- a/frontend/components/knowledge-search-bar.tsx+++ b/frontend/components/knowledge-search-bar.tsx@@ -8,12 +8,17 @@ import {} from "react";import { toast } from "sonner";import { useRefreshOpenragDocs } from "@/app/api/mutations/useRefreshOpenragDocs";-import { useSyncAllConnectors } from "@/app/api/mutations/useSyncConnector";+import {+ type SyncAllPreviewResponse,+ useSyncAllConnectors,+ useSyncAllConnectorsPreview,+} from "@/app/api/mutations/useSyncConnector";import { Button } from "@/components/ui/button";import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";import { cn } from "@/lib/utils";import { KnowledgeDropdown } from "./knowledge-dropdown";import { filterAccentClasses } from "./knowledge-filter-panel";+import { SyncConfirmDialog } from "./sync-confirm-dialog";export const KnowledgeSearchBar = () => {const {@@ -44,7 +49,51 @@ export const KnowledgeSearchBar = () => {}, [queryOverride]);const syncAllConnectorsMutation = useSyncAllConnectors();+ const syncAllPreviewMutation = useSyncAllConnectorsPreview();const refreshOpenragDocsMutation = useRefreshOpenragDocs();+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);+ const [syncPreview, setSyncPreview] = useState<SyncAllPreviewResponse | null>(+ null,+ );++ const handleOpenSyncDialog = useCallback(async () => {+ setSyncPreview(null);+ setSyncDialogOpen(true);+ try {+ const preview = await syncAllPreviewMutation.mutateAsync();+ setSyncPreview(preview);+ } catch (error) {+ setSyncDialogOpen(false);+ toast.error(+ error instanceof Error ? error.message : "Failed to preview sync",+ );+ }+ }, [syncAllPreviewMutation]);++ const handleConfirmSync = useCallback(async () => {+ try {+ const result = await syncAllConnectorsMutation.mutateAsync();+ if (result.status === "no_files") {+ toast.info(+ result.message ||+ "No cloud files to sync. Add files from cloud connectors first.",+ );+ } else if (+ result.synced_connectors &&+ result.synced_connectors.length > 0+ ) {+ toast.success(+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,+ );+ } else if (result.errors && result.errors.length > 0) {+ toast.error("Some connectors failed to sync");+ }+ } catch (error) {+ toast.error(+ error instanceof Error ? error.message : "Failed to sync connectors",+ );+ }+ }, [syncAllConnectorsMutation]);return (<form onSubmit={handleSearch} className={"flex w-full items-stretch"}>@@ -111,37 +160,14 @@ export const KnowledgeSearchBar = () => {<Buttontype="button"variant="ghost"- disabled={syncAllConnectorsMutation.isPending}+ disabled={+ syncAllConnectorsMutation.isPending ||+ syncAllPreviewMutation.isPending+ }size="icon"className="h-auto flex-shrink-0 rounded-none hover:bg-accent hover:text-foreground"aria-label="Sync"- onClick={async () => {- try {- toast.info("Syncing all cloud connectors...");- const result = await syncAllConnectorsMutation.mutateAsync();- if (result.status === "no_files") {- toast.info(- result.message ||- "No cloud files to sync. Add files from cloud connectors first.",- );- } else if (- result.synced_connectors &&- result.synced_connectors.length > 0- ) {- toast.success(- `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,- );- } else if (result.errors && result.errors.length > 0) {- toast.error("Some connectors failed to sync");- }- } catch (error) {- toast.error(- error instanceof Error- ? error.message- : "Failed to sync connectors",- );- }- }}+ onClick={handleOpenSyncDialog}><RefreshCw className="h-4 w-4 m-4 text-[var(--icon-primary)]" /></Button>@@ -172,6 +198,17 @@ export const KnowledgeSearchBar = () => {<KnowledgeDropdown /></div></div>+ <SyncConfirmDialog+ open={syncDialogOpen}+ onOpenChange={setSyncDialogOpen}+ onConfirm={handleConfirmSync}+ isLoading={syncAllPreviewMutation.isPending || syncPreview === null}+ isSyncing={syncAllConnectorsMutation.isPending}+ isSyncAll+ orphansByType={syncPreview?.orphans_by_type}+ orphansAvailableByType={syncPreview?.orphans_available_by_type}+ syncedCountByType={syncPreview?.synced_count_by_type}+ /></form>);};diff --git a/frontend/components/sync-confirm-dialog.tsx b/frontend/components/sync-confirm-dialog.tsxnew file mode 100644index 000000000..4d1b76308--- /dev/null+++ b/frontend/components/sync-confirm-dialog.tsx@@ -0,0 +1,229 @@+"use client";++import { AlertTriangle, RefreshCw } from "lucide-react";+import type React from "react";+import type { OrphanFile } from "@/app/api/mutations/useSyncConnector";+import { Badge } from "./ui/badge";+import { Button } from "./ui/button";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "./ui/dialog";+import { ScrollArea } from "./ui/scroll-area";+import { Separator } from "./ui/separator";++const CONNECTOR_DISPLAY_NAMES: Record<string, string> = {+ google_drive: "Google Drive",+ onedrive: "OneDrive",+ sharepoint: "SharePoint",+ ibm_cos: "IBM Cloud Object Storage",+ aws_s3: "Amazon S3",+};++const formatConnectorLabel = (type: string): string =>+ CONNECTOR_DISPLAY_NAMES[type] ?? type;++interface SyncConfirmDialogProps {+ open: boolean;+ onOpenChange: (open: boolean) => void;+ onConfirm: () => void | Promise<void>;+ /** True while the preview request is still in flight. */+ isLoading?: boolean;+ /** True after Confirm is clicked, while the actual sync request runs. */+ isSyncing?: boolean;+ /** Single-connector mode: the orphan list for this connector. */+ orphans?: OrphanFile[];+ /** Sync-all mode: orphans grouped by connector_type. */+ orphansByType?: Record<string, OrphanFile[]>;+ /** Per-connector availability flag — false means orphan detection couldn't+ * complete safely (e.g. unauthenticated connection). */+ orphansAvailableByType?: Record<string, boolean>;+ /** Single-connector mode: total files about to be re-synced. */+ syncedCount?: number;+ /** Sync-all mode: per-connector synced counts. */+ syncedCountByType?: Record<string, number>;+ /** Single-connector mode: connector type for the title (e.g. "sharepoint"). */+ connectorType?: string;+ /** When true, render the sync-all view (groups by connector_type). */+ isSyncAll?: boolean;+}++export const SyncConfirmDialog: React.FC<SyncConfirmDialogProps> = ({+ open,+ onOpenChange,+ onConfirm,+ isLoading = false,+ isSyncing = false,+ orphans,+ orphansByType,+ orphansAvailableByType,+ syncedCount,+ syncedCountByType,+ connectorType,+ isSyncAll = false,+}) => {+ const handleConfirm = async () => {+ await onConfirm();+ onOpenChange(false);+ };++ const totalOrphans = isSyncAll+ ? Object.values(orphansByType ?? {}).reduce(+ (sum, list) => sum + list.length,+ 0,+ )+ : (orphans?.length ?? 0);++ const totalSynced = isSyncAll+ ? Object.values(syncedCountByType ?? {}).reduce((sum, n) => sum + n, 0)+ : (syncedCount ?? 0);++ const hasOrphans = totalOrphans > 0;+ const busy = isLoading || isSyncing;++ const title = isSyncAll ? "Sync all connectors" : "Confirm sync";++ let description: React.ReactNode;+ if (isLoading) {+ description = "Checking what will change…";+ } else if (hasOrphans) {+ description = (+ <>+ <span className="font-medium text-foreground">+ {totalOrphans} {totalOrphans === 1 ? "file" : "files"}+ </span>{" "}+ will be removed from your knowledge base because they no longer exist at+ the source. {totalSynced} {totalSynced === 1 ? "file" : "files"} will be+ re-synced. This can't be undone.+ </>+ );+ } else {+ description = (+ <>+ {totalSynced} {totalSynced === 1 ? "file" : "files"} will be re-synced.+ No files will be deleted.+ </>+ );+ }++ const renderOrphanList = (list: OrphanFile[]) => (+ <ul className="space-y-1 text-sm">+ {list.map((o) => (+ <li+ key={o.document_id}+ className="truncate text-muted-foreground"+ title={o.filename || o.document_id}+ >+ {o.filename || o.document_id}+ </li>+ ))}+ </ul>+ );++ return (+ <Dialog open={open} onOpenChange={onOpenChange}>+ <DialogContent className="sm:max-w-[560px]">+ <DialogHeader>+ <div className="flex items-center justify-between gap-2">+ <DialogTitle>+ {title}+ {!isSyncAll && connectorType ? (+ <span className="text-muted-foreground font-normal">+ {" "}+ · {formatConnectorLabel(connectorType)}+ </span>+ ) : null}+ </DialogTitle>+ {hasOrphans ? (+ <Badge+ variant="secondary"+ className="bg-accent-amber-foreground/15 text-accent-amber-foreground"+ >+ {totalOrphans} will be deleted+ </Badge>+ ) : null}+ </div>+ <DialogDescription className="pt-2 text-muted-foreground">+ {description}+ </DialogDescription>+ </DialogHeader>++ {hasOrphans ? (+ <div className="rounded-md border border-accent-amber-foreground/30 bg-accent-amber-foreground/5 p-3">+ <div className="flex items-center gap-2 mb-2 text-sm font-medium text-accent-amber-foreground">+ <AlertTriangle className="h-4 w-4" />+ Files to be deleted+ </div>+ <ScrollArea className="max-h-60">+ {isSyncAll ? (+ <div className="space-y-3 pr-2">+ {Object.entries(orphansByType ?? {})+ .filter(([, list]) => list.length > 0)+ .map(([type, list], index, arr) => (+ <div key={type}>+ <div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground mb-1">+ {formatConnectorLabel(type)} ({list.length})+ </div>+ {renderOrphanList(list)}+ {index < arr.length - 1 ? (+ <Separator className="mt-3" />+ ) : null}+ </div>+ ))}+ </div>+ ) : (+ <div className="pr-2">{renderOrphanList(orphans ?? [])}</div>+ )}+ </ScrollArea>+ </div>+ ) : null}++ {/* Surface "couldn't determine deletions" when strict gating aborted. */}+ {isSyncAll && orphansAvailableByType+ ? Object.entries(orphansAvailableByType)+ .filter(([, available]) => !available)+ .map(([type]) => (+ <div+ key={type}+ className="text-xs text-muted-foreground italic"+ >+ Couldn't determine deletions for {formatConnectorLabel(type)}{" "}+ (a connection may need re-authentication).+ </div>+ ))+ : null}++ <DialogFooter className="flex-row gap-2 justify-end">+ <Button+ type="button"+ variant="ghost"+ onClick={() => onOpenChange(false)}+ disabled={busy}+ size="sm"+ >+ Cancel+ </Button>+ <Button+ type="button"+ variant="default"+ size="sm"+ onClick={handleConfirm}+ disabled={busy}+ className={+ hasOrphans+ ? "flex items-center gap-2 !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"+ : "flex items-center gap-2"+ }+ >+ <RefreshCw className="h-3.5 w-3.5" />+ {hasOrphans ? "Delete & sync" : "Confirm sync"}+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ );+};diff --git a/src/api/connectors.py b/src/api/connectors.pyindex e4213d828..3325a5c89 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1,20 +1,20 @@-from typing import Any, Dict, List, Optional+from typing import Anyfrom fastapi import Depends, Request-from pydantic import BaseModelfrom fastapi.responses import JSONResponse, PlainTextResponse-from connectors.sharepoint.utils import is_valid_sharepoint_url+from pydantic import BaseModel+from config.settings import get_index_name-from utils.logging_config import get_logger-from utils.telemetry import TelemetryClient, Category, MessageId+from connectors.sharepoint.utils import is_valid_sharepoint_urlfrom dependencies import (get_connector_service,get_current_user,- get_rbac_service,get_session_manager,require_permission,)from session_manager import User+from utils.logging_config import get_logger+from utils.telemetry import Category, MessageId, TelemetryClientlogger = get_logger(__name__)@@ -77,16 +77,196 @@ async def get_synced_file_ids_for_connector(return [], []+async def get_synced_id_to_filename_map(+ connector_type: str,+ user_id: str,+ session_manager,+ jwt_token: str | None = None,+) -> dict[str, str]:+ """Return a {document_id: filename} map for files ingested under this connector_type.++ Uses a sub-aggregation so each document_id is paired with its top filename in+ a single OpenSearch round trip.+ """+ try:+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)++ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": connector_type}},+ "aggs": {+ "by_document_id": {+ "terms": {"field": "document_id", "size": 10000},+ "aggs": {+ "top_filename": {"terms": {"field": "filename", "size": 1}},+ },+ }+ },+ }++ result = await opensearch_client.search(index=get_index_name(), body=query_body)+ buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])++ mapping: dict[str, str] = {}+ for bucket in buckets:+ doc_id = bucket.get("key")+ if not doc_id:+ continue+ fn_buckets = bucket.get("top_filename", {}).get("buckets", [])+ mapping[doc_id] = fn_buckets[0]["key"] if fn_buckets else ""+ return mapping+ except Exception as e:+ logger.error(+ "Failed to build id→filename map",+ connector_type=connector_type,+ error=str(e),+ )+ return {}+++async def compute_orphans_for_connector_type(+ connector_type: str,+ user_id: str,+ connector_service,+ session_manager,+ jwt_token: str | None,+ existing_file_ids: list[str],+ id_to_filename: dict[str, str] | None = None,+) -> list[dict[str, str]] | None:+ """Compute orphan documents (ingested but no longer present at the source)+ for this connector_type without deleting them.++ Returns a list of {"document_id", "filename"} dicts. Returns None when strict+ gating aborts the pass (unauthenticated connection or listing exception) so+ callers can distinguish "no orphans" from "could not determine safely".+ """+ if not existing_file_ids:+ return []++ connections = await connector_service.connection_manager.list_connections(+ user_id=user_id, connector_type=connector_type+ )+ active = [c for c in connections if c.is_active]+ if not active:+ logger.info(+ "Skipping orphan compute — no active connections",+ connector_type=connector_type,+ )+ return None++ remote_ids: set = set()+ for conn in active:+ try:+ connector = await connector_service.get_connector(conn.connection_id)+ if not connector or not connector.is_authenticated:+ logger.info(+ "Skipping orphan compute — connection unauthenticated",+ connector_type=connector_type,+ connection_id=conn.connection_id,+ )+ return None+ page_token = None+ while True:+ page = await connector.list_files(page_token=page_token)+ for f in page.get("files", []):+ fid = f.get("id")+ if fid:+ remote_ids.add(fid)+ page_token = page.get("nextPageToken") or page.get("next_page_token")+ if not page_token:+ break+ except Exception as e:+ logger.warning(+ "Skipping orphan compute — listing failed",+ connector_type=connector_type,+ connection_id=conn.connection_id,+ error=str(e),+ )+ return None++ orphan_ids = [fid for fid in existing_file_ids if fid not in remote_ids]+ if not orphan_ids:+ return []++ fn_map = id_to_filename or {}+ return [{"document_id": fid, "filename": fn_map.get(fid, "")} for fid in orphan_ids]+++async def delete_orphan_documents(+ orphan_ids: list[str],+ user_id: str,+ session_manager,+ jwt_token: str | None,+) -> int:+ """Delete OpenSearch chunks for the given orphan document IDs. Returns the+ number of chunks deleted (0 on failure)."""+ if not orphan_ids:+ return 0+ from .documents import delete_chunks_by_document_ids++ try:+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)+ return await delete_chunks_by_document_ids(orphan_ids, opensearch_client, get_index_name())+ except Exception as e:+ logger.error(+ "Orphan delete failed",+ orphan_count=len(orphan_ids),+ error=str(e),+ )+ return 0+++async def reconcile_orphans_for_connector_type(+ connector_type: str,+ user_id: str,+ connector_service,+ session_manager,+ jwt_token: str | None,+ existing_file_ids: list[str],+) -> list[str]:+ """Compute and delete orphans for a connector type. Thin wrapper around+ compute_orphans_for_connector_type + delete_orphan_documents preserved for+ callers that perform sync immediately after reconcile.++ Returns the list of orphan file IDs that were deleted (or []).+ """+ orphans = await compute_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ )+ if not orphans:+ return []++ orphan_ids = [o["document_id"] for o in orphans]+ deleted = await delete_orphan_documents(+ orphan_ids=orphan_ids,+ user_id=user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )+ logger.info(+ "Orphan reconcile complete",+ connector_type=connector_type,+ orphan_count=len(orphan_ids),+ deleted_chunks=deleted,+ )+ return orphan_ids++class ConnectorSyncBody(BaseModel):- max_files: Optional[int] = None- selected_files: Optional[List[Any]] = None+ max_files: int | None = None+ selected_files: list[Any] | None = None# When True, ingest ALL files from the connector (bypasses the existing-files gate).# Used by direct-sync providers like IBM COS on initial ingest.sync_all: bool = False# When set, only ingest files from these buckets (IBM COS specific).- bucket_filter: Optional[List[str]] = None+ bucket_filter: list[str] | None = None# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).- settings: Optional[Dict[str, Any]] = None+ settings: dict[str, Any] | None = Noneasync def list_connectors(@@ -279,6 +459,18 @@ async def connector_sync(connector_type=connector_type,file_count=len(existing_file_ids),)+ # Reconcile orphans (files deleted at the source) before re-syncing.+ # Strict gating: skip when sync is capped — we'd see a partial remote+ # listing and delete legitimate files.+ if body.max_files is None:+ await reconcile_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ )task_id = await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,@@ -358,7 +550,7 @@ async def connector_status()else:logger.debug(- f"connector_status: Connector has no base_url or sharepoint_url attribute"+ "connector_status: Connector has no base_url or sharepoint_url attribute")connection_details[connection.connection_id] = {@@ -739,6 +931,17 @@ async def sync_all_connectors(connector_type=connector_type,file_count=len(existing_file_ids),)+ # Reconcile orphans (files deleted at the source) before re-syncing.+ # sync_all_connectors has no caps or filters, so gating reduces+ # to the strict checks inside the helper.+ await reconcile_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ )task_id = await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,@@ -817,6 +1020,137 @@ async def sync_all_connectors(return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)+CLOUD_CONNECTOR_TYPES = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]+++async def _preview_orphans_for_connector_type(+ connector_type: str,+ user_id: str,+ connector_service,+ session_manager,+ jwt_token: str | None,+) -> tuple[list[dict[str, str]] | None, int]:+ """Helper: compute orphans (no deletion) + return total synced count.++ Returns (orphans, synced_count). `orphans` is None when strict gating aborts+ (so the caller can surface a "couldn't determine" state); [] when no orphans.+ """+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )++ synced_count = len(existing_file_ids) if existing_file_ids else len(existing_filenames)+ if not existing_file_ids:+ # No document_ids to diff against (e.g. Langflow-only ingest). Filename-only+ # fallback can't detect orphans safely — surface empty list.+ return [], synced_count++ id_to_filename = await get_synced_id_to_filename_map(+ connector_type=connector_type,+ user_id=user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )++ orphans = await compute_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ id_to_filename=id_to_filename,+ )+ return orphans, synced_count+++async def connector_sync_preview(+ connector_type: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(require_permission("connectors:use")),+):+ """Preview the impact of syncing a connector type without performing any+ deletion or ingest. Returns the list of orphan files (present in OpenSearch+ but no longer at the source) by filename, plus the total synced count.+ """+ try:+ orphans, synced_count = await _preview_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=user.jwt_token,+ )+ return JSONResponse(+ {+ "connector_type": connector_type,+ "synced_count": synced_count,+ "orphans": orphans or [],+ "orphans_available": orphans is not None,+ },+ status_code=200,+ )+ except Exception as e:+ logger.error("Sync preview failed", connector_type=connector_type, error=str(e))+ return JSONResponse({"error": f"Sync preview failed: {str(e)}"}, status_code=500)+++async def connectors_sync_all_preview(+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(require_permission("connectors:use")),+):+ """Preview the impact of sync-all-connectors across every cloud connector+ type. Returns orphan filenames grouped by connector_type plus a per-type+ synced count.+ """+ try:+ orphans_by_type: dict[str, list[dict[str, str]]] = {}+ synced_count_by_type: dict[str, int] = {}+ orphans_available_by_type: dict[str, bool] = {}++ for connector_type in CLOUD_CONNECTOR_TYPES:+ try:+ orphans, synced_count = await _preview_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=user.jwt_token,+ )+ except Exception as e:+ logger.warning(+ "Sync-all preview: per-connector failure",+ connector_type=connector_type,+ error=str(e),+ )+ orphans, synced_count = None, 0++ # Only include connector types that have something synced.+ if synced_count == 0 and not orphans:+ continue++ synced_count_by_type[connector_type] = synced_count+ orphans_by_type[connector_type] = orphans or []+ orphans_available_by_type[connector_type] = orphans is not None++ return JSONResponse(+ {+ "orphans_by_type": orphans_by_type,+ "synced_count_by_type": synced_count_by_type,+ "orphans_available_by_type": orphans_available_by_type,+ },+ status_code=200,+ )+ except Exception as e:+ logger.error("Sync-all preview failed", error=str(e))+ return JSONResponse({"error": f"Sync-all preview failed: {str(e)}"}, status_code=500)++async def connector_token(connector_type: str,connection_id: str,@@ -942,9 +1276,9 @@ async def browse_connection_files(connector_service=Depends(get_connector_service),session_manager=Depends(get_session_manager),user: User = Depends(get_current_user),- bucket: Optional[str] = None,- search: Optional[str] = None,- page_token: Optional[str] = None,+ bucket: str | None = None,+ search: str | None = None,+ page_token: str | None = None,max_files: int = 100,):"""diff --git a/src/api/documents.py b/src/api/documents.pyindex ce9b6275b..421f966fe 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -1,10 +1,10 @@from fastapi import Depends-from pydantic import BaseModelfrom fastapi.responses import JSONResponse-from utils.logging_config import get_logger+from pydantic import BaseModel-from dependencies import get_session_manager, get_current_user, require_permission+from dependencies import get_current_user, get_session_manager, require_permissionfrom session_manager import User+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -122,10 +122,24 @@ async def delete_documents_by_filename_core()+async def delete_chunks_by_document_ids(+ document_ids: list[str],+ opensearch_client,+ index_name: str,+) -> int:+ """Bulk delete OpenSearch chunks by document_id. Returns deleted count."""+ if not document_ids:+ return 0+ body = {"query": {"terms": {"document_id": document_ids}}}+ res = await opensearch_client.delete_by_query(index=index_name, body=body, conflicts="proceed")+ return res.get("deleted", 0)++async def _ensure_index_exists(jwt_token: str = None):"""Create the OpenSearch index if it doesn't exist yet."""+ from config.settings import IBM_AUTH_ENABLED+ from config.settings import clients as app_clientsfrom main import init_index- from config.settings import IBM_AUTH_ENABLED, clients as app_clientsopensearch_client = Noneif IBM_AUTH_ENABLED and jwt_token:@@ -147,8 +161,8 @@ async def check_filename_exists(try:opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)- from utils.opensearch_queries import build_filename_search_bodyfrom utils.file_utils import get_filename_aliases+ from utils.opensearch_queries import build_filename_search_bodycandidate_filenames = get_filename_aliases(filename)if not candidate_filenames:diff --git a/src/app/routes/internal.py b/src/app/routes/internal.pyindex 89072bd84..dd44b3d11 100644--- a/src/app/routes/internal.py+++ b/src/app/routes/internal.py@@ -239,12 +239,24 @@ def register_internal_routes(app: FastAPI):methods=["POST"],tags=["internal"],)+ app.add_api_route(+ "/connectors/{connector_type}/sync-preview",+ connectors.connector_sync_preview,+ methods=["POST"],+ tags=["internal"],+ )app.add_api_route("/connectors/sync-all",connectors.sync_all_connectors,methods=["POST"],tags=["internal"],)+ app.add_api_route(+ "/connectors/sync-all-preview",+ connectors.connectors_sync_all_preview,+ methods=["POST"],+ tags=["internal"],+ )app.add_api_route("/connectors/{connector_type}/status",connectors.connector_status,diff --git a/src/connectors/service.py b/src/connectors/service.pyindex fe463f590..146c70116 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -1,11 +1,10 @@-from typing import Any, Dict, List, Optional+from typing import Any+from utils.file_utils import clean_connector_filename, get_file_extensionfrom utils.logging_config import get_loggerfrom .base import BaseConnector, ConnectorDocumentfrom .connection_manager import ConnectionManager-from utils.file_utils import get_file_extension, clean_connector_filename-logger = get_logger(__name__)@@ -38,7 +37,7 @@ class ConnectorService:"""Initialize the service by loading existing connections"""await self.connection_manager.load_connections()- async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:+ async def get_connector(self, connection_id: str) -> BaseConnector | None:"""Get a connector by connection ID"""return await self.connection_manager.get_connector(connection_id)@@ -50,7 +49,7 @@ class ConnectorService:jwt_token: str = None,owner_name: str = None,owner_email: str = None,- ) -> Dict[str, Any]:+ ) -> dict[str, Any]:"""Process a document from a connector using existing processing pipeline"""# Create temporary file from document content@@ -153,6 +152,9 @@ class ConnectorService:"source": """ctx._source.source_url = params.source_url;ctx._source.connector_type = params.connector_type;+ if (params.filename != null) {+ ctx._source.filename = params.filename;+ }if (params.created_time != null) {ctx._source.created_time = params.created_time;}@@ -166,6 +168,7 @@ class ConnectorService:"params": {"source_url": document.source_url,"connector_type": connector_type,+ "filename": document.filename,"created_time": document.created_time.isoformat()if document.created_timeelse None,@@ -227,7 +230,7 @@ class ConnectorService:raise ValueError(f"Connection '{connection_id}' not authenticated")# Collect files to process (limited by max_files)- files_to_process = []+ files_to_process: list[dict[str, Any]] = []page_token = None# Calculate page size to minimize API calls@@ -236,7 +239,7 @@ class ConnectorService:while True:# List files from connector with limitlogger.debug("Calling list_files", page_size=page_size, page_token=page_token)- file_list = await connector.list_files(page_token, limit=page_size)+ file_list = await connector.list_files(page_token, max_files=page_size)logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))files = file_list["files"]@@ -311,10 +314,10 @@ class ConnectorService:self,connection_id: str,user_id: str,- file_ids: List[str],+ file_ids: list[str],jwt_token: str = None,- file_infos: List[Dict[str, Any]] = None,- ingest_settings: Optional[Dict[str, Any]] = None,+ file_infos: list[dict[str, Any]] = None,+ ingest_settings: dict[str, Any] | None = None,) -> str:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).@@ -370,8 +373,8 @@ class ConnectorService:try:# Set the file_ids we want to sync in the connector's configif hasattr(connector, "cfg"):- connector.cfg.file_ids = file_ids # type: ignore- connector.cfg.folder_ids = None # type: ignore+ connector.cfg.file_ids = file_ids+ connector.cfg.folder_ids = None# Get the expanded list of file IDs (folders will be expanded to their contents)# This uses the connector's list_files() which calls _iter_selected_items()@@ -416,8 +419,8 @@ class ConnectorService:finally:# Restore original config valuesif hasattr(connector, "cfg"):- connector.cfg.file_ids = original_file_ids # type: ignore- connector.cfg.folder_ids = original_folder_ids # type: ignore+ connector.cfg.file_ids = original_file_ids+ connector.cfg.folder_ids = original_folder_ids# Create custom processor for specific connector filesfrom models.processors import ConnectorFileProcessor@@ -456,6 +459,6 @@ class ConnectorService:return task_id- async def _get_connector(self, connection_id: str) -> Optional[BaseConnector]:+ async def _get_connector(self, connection_id: str) -> BaseConnector | None:"""Get a connector by connection ID (alias for get_connector)"""return await self.get_connector(connection_id)diff --git a/src/models/processors.py b/src/models/processors.pyindex d9e225d2b..01df83a88 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -1,13 +1,28 @@+import asyncio+import datetime+import mimetypes+import os+import timefrom typing import TYPE_CHECKING, Any+from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config+from utils.document_processing import (+ extract_relevant,+ process_text_file,+ resplit_chunks_character_windows,+)+from utils.embedding_fields import ensure_embedding_field_existsfrom utils.file_utils import (+ auto_cleanup_tempfile,clean_connector_filename,get_file_extension,get_filename_aliases,)+from utils.hash_utils import hash_idfrom utils.logging_config import get_logger+from utils.opensearch_queries import build_filename_delete_body, build_filename_search_body-from .tasks import FileTask, UploadTask+from .tasks import FileTask, TaskStatus, UploadTasklogger = get_logger(__name__)@@ -32,10 +47,6 @@ class TaskProcessor:Check if a document with the given hash already exists in OpenSearch.Consolidated hash checking for all processors."""- import asyncio-- from config.settings import get_index_name-max_retries = 3retry_delay = 1.0@@ -78,11 +89,6 @@ class TaskProcessor:Check if a document with the given filename already exists in OpenSearch.Returns True if any chunks with this filename exist."""- import asyncio-- from config.settings import get_index_name- from utils.opensearch_queries import build_filename_search_body-max_retries = 3retry_delay = 1.0@@ -149,9 +155,6 @@ class TaskProcessor:"""Delete all chunks of a document with the given filename from OpenSearch."""- from config.settings import get_index_name- from utils.opensearch_queries import build_filename_delete_body-try:deleted_count = 0candidate_filenames = get_filename_aliases(filename)@@ -204,20 +207,7 @@ class TaskProcessor:chunk_overlap: Overlap between windows; must be less than ``chunk_size``.acl: DocumentACL instance with access control information"""- import datetime-- from config.settings import (- clients,- get_embedding_model,- get_index_name,- get_openrag_config,- )from services.document_service import chunk_texts_for_embeddings- from utils.document_processing import (- extract_relevant,- resplit_chunks_character_windows,- )- from utils.embedding_fields import ensure_embedding_field_exists# Use provided embedding model or configured model.# get_embedding_model() returns empty string when Langflow ingest is enabled,@@ -241,14 +231,10 @@ class TaskProcessor:)# Check if this is a .txt or .md file - use simple processing instead of docling- import os-file_ext = os.path.splitext(file_path)[1].lower()if file_ext in (".txt", ".md"):# Simple text file processing without docling- from utils.document_processing import process_text_file-logger.info("Processing as plain text file (bypassing docling)",file_path=file_path,@@ -418,12 +404,6 @@ class DocumentFileProcessor(TaskProcessor):async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a regular file path using consolidated methods"""- import os- import time-- from models.tasks import TaskStatus- from utils.hash_utils import hash_id-file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -499,11 +479,6 @@ class ConnectorFileProcessor(TaskProcessor):async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using consolidated methods"""- import time-- from models.tasks import TaskStatus- from utils.hash_utils import hash_id-file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -519,7 +494,23 @@ class ConnectorFileProcessor(TaskProcessor):raise ValueError(f"Connection '{self.connection_id}' not found")# Get file content from connector- document = await connector.get_file_content(file_id)+ try:+ document = await connector.get_file_content(file_id)+ except (FileNotFoundError, ValueError) as e:+ msg = str(e).lower()+ if "not found" in msg or "404" in msg:+ logger.warning(+ "File no longer exists at source — skipping",+ file_id=file_id,+ connection_id=self.connection_id,+ error=str(e),+ )+ file_task.status = TaskStatus.SKIPPED+ file_task.result = {"status": "skipped", "reason": "deleted_at_source"}+ file_task.updated_at = time.time()+ upload_task.successful_files += 1+ return+ raise# Update filename in task once we have it from the connectorfile_task.filename = clean_connector_filename(document.filename, document.mimetype)@@ -528,8 +519,6 @@ class ConnectorFileProcessor(TaskProcessor):raise ValueError("user_id not provided to ConnectorFileProcessor")# Create temporary file from document content- from utils.file_utils import auto_cleanup_tempfile-suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp file@@ -626,11 +615,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using LangflowConnectorService"""- import time-- from models.tasks import TaskStatus- from utils.hash_utils import hash_id-file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()@@ -655,8 +639,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):raise ValueError("user_id not provided to LangflowConnectorFileProcessor")# Create temporary file and compute hash to check for duplicates- from utils.file_utils import auto_cleanup_tempfile-suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp file@@ -734,19 +716,9 @@ class S3FileProcessor(TaskProcessor):async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Download an S3 object and process it using DocumentService"""- import time- import asyncio- import datetime- from config.settings import clients, get_embedding_model, get_index_name-- from models.tasks import TaskStatus-file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()- from utils.file_utils import auto_cleanup_tempfile- from utils.hash_utils import hash_id-try:with auto_cleanup_tempfile() as tmp_path:# Download object to temporary file@@ -827,12 +799,6 @@ class LangflowFileProcessor(TaskProcessor):async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a file path using LangflowFileService upload_and_ingest_file"""- import mimetypes- import os- import time-- from models.tasks import TaskStatus-# Update task statusfile_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()diff --git a/src/models/tasks.py b/src/models/tasks.pyindex 40bda40ff..747579f62 100644--- a/src/models/tasks.py+++ b/src/models/tasks.py@@ -2,7 +2,7 @@ import itertoolsimport timefrom dataclasses import dataclass, fieldfrom enum import Enum-from typing import ClassVar, Dict, Optional+from typing import ClassVarclass TaskStatus(Enum):@@ -10,6 +10,7 @@ class TaskStatus(Enum):RUNNING = "running"COMPLETED = "completed"FAILED = "failed"+ SKIPPED = "skipped"class DoclingPhaseStatus(Enum):@@ -40,17 +41,17 @@ class IngestionPhase(Enum):class FileTask:file_path: strstatus: TaskStatus = TaskStatus.PENDING- result: Optional[dict] = None- error: Optional[str] = None+ result: dict | None = None+ error: str | None = Noneretry_count: int = 0created_at: float = field(default_factory=time.time)updated_at: float = field(default_factory=time.time)- filename: Optional[str] = None # Original filename for display# Two-phase ingestion fields. Only meaningful for processors that submit# files to Docling Serve and then trigger Langflow (i.e. LangflowFileProcessor).- docling_task_id: Optional[str] = None+ docling_task_id: str | None = Nonedocling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDINGphase: IngestionPhase = IngestionPhase.DOCLING+ filename: str | None = None # Original filename for display@propertydef duration_seconds(self) -> float:@@ -67,7 +68,7 @@ class UploadTask:processed_files: int = 0successful_files: int = 0failed_files: int = 0- file_tasks: Dict[str, FileTask] = field(default_factory=dict)+ file_tasks: dict[str, FileTask] = field(default_factory=dict)status: TaskStatus = TaskStatus.PENDINGcreated_at: float = field(default_factory=time.time)updated_at: float = field(default_factory=time.time)diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.pynew file mode 100644index 000000000..005a1477d--- /dev/null+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py@@ -0,0 +1,62 @@+"""Unit tests for the bulk-delete helper used by the orphan-reconcile pass.++Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:+- empty input is a no-op (no OpenSearch call)+- non-empty input issues a single delete_by_query with terms(document_id, ...)+- returns the count reported by OpenSearch+"""++import sys+from pathlib import Path+from unittest.mock import AsyncMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++async def test_empty_ids_short_circuits_without_calling_opensearch():+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = AsyncMock()+ deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_issues_single_delete_by_query_with_terms_filter():+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = AsyncMock()+ opensearch_client.delete_by_query.return_value = {"deleted": 12}++ ids = ["abc", "def", "ghi"]+ deleted = await delete_chunks_by_document_ids(ids, opensearch_client, "test-index")++ assert deleted == 12+ opensearch_client.delete_by_query.assert_awaited_once()+ call = opensearch_client.delete_by_query.await_args+ assert call.kwargs["index"] == "test-index"+ assert call.kwargs["body"] == {"query": {"terms": {"document_id": ids}}}+ # conflicts="proceed" so a concurrent reindex doesn't abort the bulk delete.+ assert call.kwargs.get("conflicts") == "proceed"+++async def test_returns_zero_when_response_missing_deleted_field():+ """Defensive: if OpenSearch returns an unexpected payload shape, we+ should not crash — we surface 0 deletions."""+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = AsyncMock()+ opensearch_client.delete_by_query.return_value = {} # no "deleted" key++ deleted = await delete_chunks_by_document_ids(["abc"], opensearch_client, "test-index")++ assert deleted == 0diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.pynew file mode 100644index 000000000..bd9a1a187--- /dev/null+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py@@ -0,0 +1,338 @@+"""Unit tests for `reconcile_orphans_for_connector_type` in `src/api/connectors.py`.++This is the orphan-deletion safety net for the connector sync flow. The+function must:+- Compute orphans = indexed_ids - union(remote_ids across all active+ connections of this connector_type for this user).+- Apply STRICT gating: any unauthenticated connection or listing+ exception aborts the pass with 0 deletes (false-negative > false-positive).+- Preserve files that exist in any active connection of the type+ (multi-connection isolation).+"""++import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_connection(connection_id: str, is_active: bool = True):+ """Lightweight stand-in for ConnectionConfig — only the attributes+ reconcile_orphans_for_connector_type reads."""+ return SimpleNamespace(connection_id=connection_id, is_active=is_active)+++def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):+ """Build an AsyncMock connector that reports the given file IDs from+ list_files() (single page, no pagination)."""+ connector = MagicMock()+ connector.is_authenticated = authenticated+ if raise_on_list:+ connector.list_files = AsyncMock(side_effect=RuntimeError("graph 503"))+ else:+ connector.list_files = AsyncMock(+ return_value={"files": [{"id": fid} for fid in remote_file_ids]}+ )+ return connector+++def _make_service(connections, connector_lookup):+ """Build a connector_service stub.++ `connections` is the list returned by list_connections.+ `connector_lookup` maps connection_id -> connector mock (or None).+ """+ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=connections)++ async def _get_connector(connection_id):+ return connector_lookup.get(connection_id)++ service.get_connector = AsyncMock(side_effect=_get_connector)+ return service+++def _make_session_manager(opensearch_client):+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)+ return sm+++async def test_empty_existing_file_ids_returns_zero_without_calls():+ from api.connectors import reconcile_orphans_for_connector_type++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock()+ sm = _make_session_manager(AsyncMock())++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=[],+ )++ assert deleted == 0+ service.connection_manager.list_connections.assert_not_awaited()+++async def test_no_active_connections_skips_reconcile():+ """If every connection is inactive, we have no remote view — skip."""+ from api.connectors import reconcile_orphans_for_connector_type++ inactive = [_make_connection("c1", is_active=False)]+ service = _make_service(inactive, connector_lookup={})+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_unauthenticated_connection_aborts_pass(monkeypatch):+ """STRICT GATING: even one unauthenticated connector aborts the pass.+ Otherwise we'd wrongly mark a file as an orphan because we couldn't+ list the connection that holds it."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=[], authenticated=False)+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0+ connector.list_files.assert_not_awaited()+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_listing_exception_aborts_pass():+ """STRICT GATING: a transient list_files error aborts the pass.+ Better to leave orphans for one cycle than delete legitimate files."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=[], raise_on_list=True)+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_happy_path_deletes_orphans():+ """Indexed has [a, b, c]; remote has [a, c] -> orphan = [b]."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a", "c"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = AsyncMock()+ opensearch_client.delete_by_query.return_value = {"deleted": 7}+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b", "c"],+ )++ assert deleted == 7+ opensearch_client.delete_by_query.assert_awaited_once()+ body = opensearch_client.delete_by_query.await_args.kwargs["body"]+ assert body == {"query": {"terms": {"document_id": ["b"]}}}+++async def test_no_orphans_skips_delete_call():+ """If every indexed ID still exists remotely, we must not issue an+ empty delete_by_query."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a", "b"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_multi_connection_union_preserves_files_present_in_any_connection():+ """Multi-connection isolation:+ - User has conn-A and conn-B (both SharePoint, different sites).+ - Indexed has [a, b]. conn-A has [a]. conn-B has [b].+ - Neither file is an orphan — both should be preserved."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn_a = _make_connection("conn-a")+ conn_b = _make_connection("conn-b")+ connector_a = _make_connector(remote_file_ids=["a"])+ connector_b = _make_connector(remote_file_ids=["b"])+ service = _make_service(+ [conn_a, conn_b],+ connector_lookup={"conn-a": connector_a, "conn-b": connector_b},+ )+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():+ """If conn-A lists fine but conn-B is unauthenticated, we MUST NOT+ treat files only in conn-B as orphans. Strict gating bails out."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn_a = _make_connection("conn-a")+ conn_b = _make_connection("conn-b")+ connector_a = _make_connector(remote_file_ids=["a"])+ connector_b = _make_connector(remote_file_ids=[], authenticated=False)+ service = _make_service(+ [conn_a, conn_b],+ connector_lookup={"conn-a": connector_a, "conn-b": connector_b},+ )+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"], # b would look like an orphan if we trusted conn-A alone+ )++ assert deleted == 0+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_paginated_listing_aggregates_all_pages():+ """Remote listings are paginated. The reconcile must walk every page+ before computing the orphan set, otherwise files on page 2 look like+ orphans."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = MagicMock()+ connector.is_authenticated = True+ pages = [+ {"files": [{"id": "a"}], "nextPageToken": "tok-1"},+ {"files": [{"id": "b"}, {"id": "c"}]}, # last page, no token+ ]+ connector.list_files = AsyncMock(side_effect=pages)++ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = AsyncMock()+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b", "c"],+ )++ assert deleted == 0+ assert connector.list_files.await_count == 2+ opensearch_client.delete_by_query.assert_not_awaited()+++async def test_delete_failure_does_not_raise():+ """If the bulk delete itself blows up, the helper must swallow it and+ return 0. The caller (sync) should still proceed to re-sync surviving+ files — leaving orphans is recoverable, raising is not."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a"])+ service = _make_service([conn], connector_lookup={"c1": connector})++ opensearch_client = AsyncMock()+ opensearch_client.delete_by_query.side_effect = RuntimeError("opensearch unavailable")+ sm = _make_session_manager(opensearch_client)++ deleted = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert deleted == 0diff --git a/tests/unit/connectors/__init__.py b/tests/unit/connectors/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.pynew file mode 100644index 000000000..14cbccaa2--- /dev/null+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py@@ -0,0 +1,134 @@+"""Connector rename fix: _update_connector_metadata must rewrite the indexed+`filename` field so renamed source files don't leave their old name in+OpenSearch chunks.++Pins the painless script extension in src/connectors/service.py+`_update_connector_metadata`.+"""++import sys+from datetime import datetime+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_service():+ """Build a ConnectorService whose only wired-up surface is what+ _update_connector_metadata touches: an opensearch client (via+ session_manager) and an index_name."""+ from connectors.service import ConnectorService++ service = ConnectorService.__new__(ConnectorService)+ service.session_manager = MagicMock()+ service.index_name = "test-index"++ opensearch_client = AsyncMock()+ service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)+ return service, opensearch_client+++def _make_document(filename: str = "renamed.pdf"):+ from connectors.base import ConnectorDocument, DocumentACL++ return ConnectorDocument(+ id="graph-item-id-stable",+ filename=filename,+ mimetype="application/pdf",+ content=b"",+ source_url="https://contoso.sharepoint.com/.../renamed.pdf",+ acl=DocumentACL(owner="alice"),+ modified_time=datetime(2026, 5, 7),+ created_time=datetime(2026, 5, 1),+ metadata={"site": "marketing"},+ )+++async def test_update_includes_filename_in_script_and_params(monkeypatch):+ """The painless script must reference params.filename, and params must+ carry document.filename as-is."""+ service, opensearch_client = _make_service()++ # update_document_acl is invoked first; stub it to a no-op so the test+ # focuses on the metadata update_by_query.+ async def _noop_acl(**_kwargs):+ return {"status": "unchanged"}++ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)++ document = _make_document(filename="Report-FY26.docx")+ await service._update_connector_metadata(+ document, owner_user_id="alice", connector_type="sharepoint"+ )++ opensearch_client.update_by_query.assert_awaited_once()+ call = opensearch_client.update_by_query.await_args+ body = call.kwargs["body"]+ script = body["script"]+ params = script["params"]++ assert "params.filename" in script["source"], (+ "painless script must read params.filename so renamed files get "+ "the new name written through to indexed chunks"+ )+ assert params["filename"] == "Report-FY26.docx"+ # Sanity: scope is by document_id (the stable connector ID).+ assert body["query"] == {"term": {"document_id": "graph-item-id-stable"}}+++async def test_filename_passed_raw_not_cleaned(monkeypatch):+ """process_document_standard indexes chunks with the raw document.filename+ (see src/connectors/service.py:80 + src/models/processors.py:323).+ The metadata update must use the SAME raw value or chunks drift.+ """+ service, opensearch_client = _make_service()++ async def _noop_acl(**_kwargs):+ return {"status": "unchanged"}++ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)++ # A name with a quirky extension/suffix that clean_connector_filename+ # would normalize. We pass it through verbatim.+ raw_name = "Q4 Report (final).docx"+ document = _make_document(filename=raw_name)++ await service._update_connector_metadata(+ document, owner_user_id="alice", connector_type="sharepoint"+ )++ params = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]["params"]+ assert params["filename"] == raw_name+++async def test_filename_overwritten_for_existing_indexed_chunks(monkeypatch):+ """End-to-end intent: when SharePoint rename triggers the unchanged ->+ metadata-update path, the painless update writes the new filename.+ Verified via the captured update_by_query body."""+ service, opensearch_client = _make_service()++ async def _noop_acl(**_kwargs):+ return {"status": "unchanged"}++ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)++ # Simulate the post-rename state: same ID, new filename.+ document = _make_document(filename="renamed-after-edit.pdf")+ await service._update_connector_metadata(+ document, owner_user_id="alice", connector_type="sharepoint"+ )++ body = opensearch_client.update_by_query.await_args.kwargs["body"]+ # The script must carry the new filename in its params, AND the source+ # must assign it onto _source.filename for every matching chunk.+ assert body["script"]["params"]["filename"] == "renamed-after-edit.pdf"+ assert "ctx._source.filename = params.filename" in body["script"]["source"]