Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Skip deleted connector files; return orphan IDs (#1592)

Edwin Josed2550408ee7b
diff --git a/.gitignore b/.gitignore
index 6f3ded523..ebc5100db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,7 +24,6 @@ wheels/
!src/tui/_assets/flows/*.json
!src/tui/_assets/flows/components/*.json
!frontend/*.json
-!**/.claude-plugin/*.json
.DS_Store
/config/
@@ -39,20 +38,16 @@ langflow-data/
node_modules
# AI Tools
-# Per-contributor Claude Code state is ignored, but shared skills and
-# plugin manifests (.claude/skills, .claude-plugin) are tracked.
-/.claude/*
-!/.claude/skills/
+/.claude
+/CLAUDE.md
/.gemini
/GEMINI.md
/.bob
+/AGENTS.md
documents/warmup_ocr.pdf
documents/openrag-documentation.pdf
documents/ibm_anthropic.pdf
documents/docling.pdf
/opensearch-data-new-lf
/opensearch-data2
-data/openrag.db
-data/openrag.db
*.db
-/data
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.ts
index fe2c7ef2e..e40508211 100644
--- a/frontend/app/api/mutations/useSyncConnector.ts
+++ b/frontend/app/api/mutations/useSyncConnector.ts
@@ -13,6 +13,27 @@ interface SyncResponse {
errors?: Array<{ connector_type: string; error: string }> | null;
}
+export interface OrphanFile {
+ document_id: string;
+ filename: string;
+}
+
+export interface SyncPreviewResponse {
+ connector_type: string;
+ synced_count: number;
+ orphans: OrphanFile[];
+ /** False when strict gating aborted orphan detection (e.g. an active
+ * connection was unauthenticated). UI should reflect that deletions
+ * cannot be predicted in that case. */
+ orphans_available: boolean;
+}
+
+export interface SyncAllPreviewResponse {
+ orphans_by_type: Record<string, OrphanFile[]>;
+ synced_count_by_type: Record<string, number>;
+ orphans_available_by_type: Record<string, boolean>;
+}
+
// Sync all cloud connectors
const syncAllConnectors = async (): Promise<SyncResponse> => {
const response = await fetch("/api/connectors/sync-all", {
@@ -92,3 +113,44 @@ export const useSyncConnector = () => {
},
});
};
+
+const syncConnectorPreview = async (
+ connectorType: string,
+): Promise<SyncPreviewResponse> => {
+ const response = await fetch(
+ `/api/connectors/${connectorType}/sync-preview`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ },
+ );
+
+ if (!response.ok) {
+ const error = await response.json().catch(() => ({}));
+ throw new Error(
+ error.error || `Failed to preview sync for ${connectorType}`,
+ );
+ }
+
+ return response.json();
+};
+
+const syncAllConnectorsPreview = async (): Promise<SyncAllPreviewResponse> => {
+ const response = await fetch("/api/connectors/sync-all-preview", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ });
+
+ if (!response.ok) {
+ const error = await response.json().catch(() => ({}));
+ throw new Error(error.error || "Failed to preview sync");
+ }
+
+ return response.json();
+};
+
+export const useSyncConnectorPreview = () =>
+ useMutation({ mutationFn: syncConnectorPreview });
+
+export const useSyncAllConnectorsPreview = () =>
+ useMutation({ mutationFn: syncAllConnectorsPreview });
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index a5f0a4efc..ea5f9af27 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -55,9 +55,14 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";
import IBMCOSIcon from "../../components/icons/ibm-cos-icon";
import OneDriveIcon from "../../components/icons/one-drive-logo";
import SharePointIcon from "../../components/icons/share-point-logo";
+import { SyncConfirmDialog } from "../../components/sync-confirm-dialog";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";
-import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
+import {
+ type SyncAllPreviewResponse,
+ useSyncAllConnectors,
+ useSyncAllConnectorsPreview,
+} from "../api/mutations/useSyncConnector";
/** List-files uses term filters; "*" means "any" in the UI — do not send it literally. */
function listFilesFilterParam(values?: string[]): string | undefined {
@@ -119,7 +124,51 @@ function SearchPage() {
const deleteDocumentMutation = useDeleteDocument();
const syncAllConnectorsMutation = useSyncAllConnectors();
+ const syncAllPreviewMutation = useSyncAllConnectorsPreview();
const refreshOpenragDocsMutation = useRefreshOpenragDocs();
+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);
+ const [syncPreview, setSyncPreview] = useState<SyncAllPreviewResponse | null>(
+ null,
+ );
+
+ const handleOpenSyncDialog = useCallback(async () => {
+ setSyncPreview(null);
+ setSyncDialogOpen(true);
+ try {
+ const preview = await syncAllPreviewMutation.mutateAsync();
+ setSyncPreview(preview);
+ } catch (error) {
+ setSyncDialogOpen(false);
+ toast.error(
+ error instanceof Error ? error.message : "Failed to preview sync",
+ );
+ }
+ }, [syncAllPreviewMutation]);
+
+ const handleConfirmSync = useCallback(async () => {
+ try {
+ const result = await syncAllConnectorsMutation.mutateAsync();
+ if (result.status === "no_files") {
+ toast.info(
+ result.message ||
+ "No cloud files to sync. Add files from cloud connectors first.",
+ );
+ } else if (
+ result.synced_connectors &&
+ result.synced_connectors.length > 0
+ ) {
+ toast.success(
+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
+ );
+ } else if (result.errors && result.errors.length > 0) {
+ toast.error("Some connectors failed to sync");
+ }
+ } catch (error) {
+ toast.error(
+ error instanceof Error ? error.message : "Failed to sync connectors",
+ );
+ }
+ }, [syncAllConnectorsMutation]);
useEffect(() => {
refreshTasks();
@@ -761,36 +810,14 @@ function SearchPage() {
type="button"
variant="outline"
className="rounded-lg flex-shrink-0"
- disabled={syncAllConnectorsMutation.isPending}
- onClick={async () => {
- try {
- toast.info("Syncing all cloud connectors...");
- const result = await syncAllConnectorsMutation.mutateAsync();
- if (result.status === "no_files") {
- toast.info(
- result.message ||
- "No cloud files to sync. Add files from cloud connectors first.",
- );
- } else if (
- result.synced_connectors &&
- result.synced_connectors.length > 0
- ) {
- toast.success(
- `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
- );
- } else if (result.errors && result.errors.length > 0) {
- toast.error("Some connectors failed to sync");
- }
- } catch (error) {
- toast.error(
- error instanceof Error
- ? error.message
- : "Failed to sync connectors",
- );
- }
- }}
+ disabled={
+ syncAllConnectorsMutation.isPending ||
+ syncAllPreviewMutation.isPending
+ }
+ onClick={handleOpenSyncDialog}
>
- {syncAllConnectorsMutation.isPending ? (
+ {syncAllConnectorsMutation.isPending ||
+ syncAllPreviewMutation.isPending ? (
<>
<RefreshCw className="h-4 w-4 mr-2 animate-spin" />
Syncing...
@@ -961,6 +988,18 @@ function SearchPage() {
<p className="my-2">Documents to be deleted:</p>
{formatFilesToDelete(selectedRows)}
</DeleteConfirmationDialog>
+
+ <SyncConfirmDialog
+ open={syncDialogOpen}
+ onOpenChange={setSyncDialogOpen}
+ onConfirm={handleConfirmSync}
+ isLoading={syncAllPreviewMutation.isPending || syncPreview === null}
+ isSyncing={syncAllConnectorsMutation.isPending}
+ isSyncAll
+ orphansByType={syncPreview?.orphans_by_type}
+ orphansAvailableByType={syncPreview?.orphans_available_by_type}
+ syncedCountByType={syncPreview?.synced_count_by_type}
+ />
</>
);
}
diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsx
index 62322d89e..a7a897c5c 100644
--- a/frontend/components/knowledge-actions-dropdown.tsx
+++ b/frontend/components/knowledge-actions-dropdown.tsx
@@ -5,7 +5,11 @@ import { useRouter } from "next/navigation";
import { useMemo, useState } from "react";
import { toast } from "sonner";
import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";
-import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
+import {
+ type SyncPreviewResponse,
+ useSyncConnector,
+ useSyncConnectorPreview,
+} from "@/app/api/mutations/useSyncConnector";
import { useGetConnectorsQuery } from "@/app/api/queries/useGetConnectorsQuery";
import {
DropdownMenu,
@@ -24,6 +28,7 @@ import { usePermissions } from "@/hooks/use-permissions";
import { formatFilesToDelete } from "@/lib/format-files-to-delete";
import { DeleteConfirmationDialog } from "./delete-confirmation-dialog";
import { RequirePermission } from "./require-permission";
+import { SyncConfirmDialog } from "./sync-confirm-dialog";
import { Button } from "./ui/button";
interface KnowledgeActionsDropdownProps {
@@ -46,6 +51,11 @@ export const KnowledgeActionsDropdown = ({
const [showDeleteDialog, setShowDeleteDialog] = useState(false);
const deleteDocumentMutation = useDeleteDocument();
const syncConnectorMutation = useSyncConnector();
+ const syncPreviewMutation = useSyncConnectorPreview();
+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);
+ const [syncPreview, setSyncPreview] = useState<SyncPreviewResponse | null>(
+ null,
+ );
const { data: connectors = [] } = useGetConnectorsQuery();
const router = useRouter();
@@ -81,11 +91,24 @@ export const KnowledgeActionsDropdown = ({
}
};
- const handleSync = async () => {
+ const handleOpenSyncDialog = async () => {
if (!connectorType || !isConnected) return;
+ setSyncPreview(null);
+ setSyncDialogOpen(true);
+ try {
+ const preview = await syncPreviewMutation.mutateAsync(connectorType);
+ setSyncPreview(preview);
+ } catch (error) {
+ setSyncDialogOpen(false);
+ toast.error(
+ error instanceof Error ? error.message : "Failed to preview sync",
+ );
+ }
+ };
+ const handleConfirmSync = async () => {
+ if (!connectorType) return;
try {
- toast.info(`Syncing ${connectorType}...`);
const result = await syncConnectorMutation.mutateAsync({ connectorType });
if (result.status === "no_files") {
toast.info(result.message || `No ${connectorType} files to sync.`);
@@ -129,16 +152,21 @@ export const KnowledgeActionsDropdown = ({
<div className="w-full">
<DropdownMenuItem
className="text-primary focus:text-primary cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed"
- disabled={syncConnectorMutation.isPending || !isConnected}
+ disabled={
+ syncConnectorMutation.isPending ||
+ syncPreviewMutation.isPending ||
+ !isConnected
+ }
onClick={(e) => {
if (!isConnected) {
e.preventDefault();
return;
}
- handleSync();
+ handleOpenSyncDialog();
}}
>
- {syncConnectorMutation.isPending ? (
+ {syncConnectorMutation.isPending ||
+ syncPreviewMutation.isPending ? (
<>
<RefreshCw className="h-4 w-4 mr-2 animate-spin" />
Syncing...
@@ -196,6 +224,17 @@ export const KnowledgeActionsDropdown = ({
<p className="my-2">Document to be deleted:</p>
{formatFilesToDelete([{ filename }])}
</DeleteConfirmationDialog>
+
+ <SyncConfirmDialog
+ open={syncDialogOpen}
+ onOpenChange={setSyncDialogOpen}
+ onConfirm={handleConfirmSync}
+ isLoading={syncPreviewMutation.isPending || syncPreview === null}
+ isSyncing={syncConnectorMutation.isPending}
+ connectorType={connectorType}
+ orphans={syncPreview?.orphans}
+ syncedCount={syncPreview?.synced_count}
+ />
</>
);
};
diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsx
index 03a449173..d7f7b3959 100644
--- a/frontend/components/knowledge-search-bar.tsx
+++ b/frontend/components/knowledge-search-bar.tsx
@@ -8,12 +8,17 @@ import {
} from "react";
import { toast } from "sonner";
import { useRefreshOpenragDocs } from "@/app/api/mutations/useRefreshOpenragDocs";
-import { useSyncAllConnectors } from "@/app/api/mutations/useSyncConnector";
+import {
+ type SyncAllPreviewResponse,
+ useSyncAllConnectors,
+ useSyncAllConnectorsPreview,
+} from "@/app/api/mutations/useSyncConnector";
import { Button } from "@/components/ui/button";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { cn } from "@/lib/utils";
import { KnowledgeDropdown } from "./knowledge-dropdown";
import { filterAccentClasses } from "./knowledge-filter-panel";
+import { SyncConfirmDialog } from "./sync-confirm-dialog";
export const KnowledgeSearchBar = () => {
const {
@@ -44,7 +49,51 @@ export const KnowledgeSearchBar = () => {
}, [queryOverride]);
const syncAllConnectorsMutation = useSyncAllConnectors();
+ const syncAllPreviewMutation = useSyncAllConnectorsPreview();
const refreshOpenragDocsMutation = useRefreshOpenragDocs();
+ const [syncDialogOpen, setSyncDialogOpen] = useState(false);
+ const [syncPreview, setSyncPreview] = useState<SyncAllPreviewResponse | null>(
+ null,
+ );
+
+ const handleOpenSyncDialog = useCallback(async () => {
+ setSyncPreview(null);
+ setSyncDialogOpen(true);
+ try {
+ const preview = await syncAllPreviewMutation.mutateAsync();
+ setSyncPreview(preview);
+ } catch (error) {
+ setSyncDialogOpen(false);
+ toast.error(
+ error instanceof Error ? error.message : "Failed to preview sync",
+ );
+ }
+ }, [syncAllPreviewMutation]);
+
+ const handleConfirmSync = useCallback(async () => {
+ try {
+ const result = await syncAllConnectorsMutation.mutateAsync();
+ if (result.status === "no_files") {
+ toast.info(
+ result.message ||
+ "No cloud files to sync. Add files from cloud connectors first.",
+ );
+ } else if (
+ result.synced_connectors &&
+ result.synced_connectors.length > 0
+ ) {
+ toast.success(
+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
+ );
+ } else if (result.errors && result.errors.length > 0) {
+ toast.error("Some connectors failed to sync");
+ }
+ } catch (error) {
+ toast.error(
+ error instanceof Error ? error.message : "Failed to sync connectors",
+ );
+ }
+ }, [syncAllConnectorsMutation]);
return (
<form onSubmit={handleSearch} className={"flex w-full items-stretch"}>
@@ -111,37 +160,14 @@ export const KnowledgeSearchBar = () => {
<Button
type="button"
variant="ghost"
- disabled={syncAllConnectorsMutation.isPending}
+ disabled={
+ syncAllConnectorsMutation.isPending ||
+ syncAllPreviewMutation.isPending
+ }
size="icon"
className="h-auto flex-shrink-0 rounded-none hover:bg-accent hover:text-foreground"
aria-label="Sync"
- onClick={async () => {
- try {
- toast.info("Syncing all cloud connectors...");
- const result = await syncAllConnectorsMutation.mutateAsync();
- if (result.status === "no_files") {
- toast.info(
- result.message ||
- "No cloud files to sync. Add files from cloud connectors first.",
- );
- } else if (
- result.synced_connectors &&
- result.synced_connectors.length > 0
- ) {
- toast.success(
- `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
- );
- } else if (result.errors && result.errors.length > 0) {
- toast.error("Some connectors failed to sync");
- }
- } catch (error) {
- toast.error(
- error instanceof Error
- ? error.message
- : "Failed to sync connectors",
- );
- }
- }}
+ onClick={handleOpenSyncDialog}
>
<RefreshCw className="h-4 w-4 m-4 text-[var(--icon-primary)]" />
</Button>
@@ -172,6 +198,17 @@ export const KnowledgeSearchBar = () => {
<KnowledgeDropdown />
</div>
</div>
+ <SyncConfirmDialog
+ open={syncDialogOpen}
+ onOpenChange={setSyncDialogOpen}
+ onConfirm={handleConfirmSync}
+ isLoading={syncAllPreviewMutation.isPending || syncPreview === null}
+ isSyncing={syncAllConnectorsMutation.isPending}
+ isSyncAll
+ orphansByType={syncPreview?.orphans_by_type}
+ orphansAvailableByType={syncPreview?.orphans_available_by_type}
+ syncedCountByType={syncPreview?.synced_count_by_type}
+ />
</form>
);
};
diff --git a/frontend/components/sync-confirm-dialog.tsx b/frontend/components/sync-confirm-dialog.tsx
new file mode 100644
index 000000000..4d1b76308
--- /dev/null
+++ b/frontend/components/sync-confirm-dialog.tsx
@@ -0,0 +1,229 @@
+"use client";
+
+import { AlertTriangle, RefreshCw } from "lucide-react";
+import type React from "react";
+import type { OrphanFile } from "@/app/api/mutations/useSyncConnector";
+import { Badge } from "./ui/badge";
+import { Button } from "./ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "./ui/dialog";
+import { ScrollArea } from "./ui/scroll-area";
+import { Separator } from "./ui/separator";
+
+const CONNECTOR_DISPLAY_NAMES: Record<string, string> = {
+ google_drive: "Google Drive",
+ onedrive: "OneDrive",
+ sharepoint: "SharePoint",
+ ibm_cos: "IBM Cloud Object Storage",
+ aws_s3: "Amazon S3",
+};
+
+const formatConnectorLabel = (type: string): string =>
+ CONNECTOR_DISPLAY_NAMES[type] ?? type;
+
+interface SyncConfirmDialogProps {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ onConfirm: () => void | Promise<void>;
+ /** True while the preview request is still in flight. */
+ isLoading?: boolean;
+ /** True after Confirm is clicked, while the actual sync request runs. */
+ isSyncing?: boolean;
+ /** Single-connector mode: the orphan list for this connector. */
+ orphans?: OrphanFile[];
+ /** Sync-all mode: orphans grouped by connector_type. */
+ orphansByType?: Record<string, OrphanFile[]>;
+ /** Per-connector availability flag — false means orphan detection couldn't
+ * complete safely (e.g. unauthenticated connection). */
+ orphansAvailableByType?: Record<string, boolean>;
+ /** Single-connector mode: total files about to be re-synced. */
+ syncedCount?: number;
+ /** Sync-all mode: per-connector synced counts. */
+ syncedCountByType?: Record<string, number>;
+ /** Single-connector mode: connector type for the title (e.g. "sharepoint"). */
+ connectorType?: string;
+ /** When true, render the sync-all view (groups by connector_type). */
+ isSyncAll?: boolean;
+}
+
+export const SyncConfirmDialog: React.FC<SyncConfirmDialogProps> = ({
+ open,
+ onOpenChange,
+ onConfirm,
+ isLoading = false,
+ isSyncing = false,
+ orphans,
+ orphansByType,
+ orphansAvailableByType,
+ syncedCount,
+ syncedCountByType,
+ connectorType,
+ isSyncAll = false,
+}) => {
+ const handleConfirm = async () => {
+ await onConfirm();
+ onOpenChange(false);
+ };
+
+ const totalOrphans = isSyncAll
+ ? Object.values(orphansByType ?? {}).reduce(
+ (sum, list) => sum + list.length,
+ 0,
+ )
+ : (orphans?.length ?? 0);
+
+ const totalSynced = isSyncAll
+ ? Object.values(syncedCountByType ?? {}).reduce((sum, n) => sum + n, 0)
+ : (syncedCount ?? 0);
+
+ const hasOrphans = totalOrphans > 0;
+ const busy = isLoading || isSyncing;
+
+ const title = isSyncAll ? "Sync all connectors" : "Confirm sync";
+
+ let description: React.ReactNode;
+ if (isLoading) {
+ description = "Checking what will change…";
+ } else if (hasOrphans) {
+ description = (
+ <>
+ <span className="font-medium text-foreground">
+ {totalOrphans} {totalOrphans === 1 ? "file" : "files"}
+ </span>{" "}
+ will be removed from your knowledge base because they no longer exist at
+ the source. {totalSynced} {totalSynced === 1 ? "file" : "files"} will be
+ re-synced. This can't be undone.
+ </>
+ );
+ } else {
+ description = (
+ <>
+ {totalSynced} {totalSynced === 1 ? "file" : "files"} will be re-synced.
+ No files will be deleted.
+ </>
+ );
+ }
+
+ const renderOrphanList = (list: OrphanFile[]) => (
+ <ul className="space-y-1 text-sm">
+ {list.map((o) => (
+ <li
+ key={o.document_id}
+ className="truncate text-muted-foreground"
+ title={o.filename || o.document_id}
+ >
+ {o.filename || o.document_id}
+ </li>
+ ))}
+ </ul>
+ );
+
+ return (
+ <Dialog open={open} onOpenChange={onOpenChange}>
+ <DialogContent className="sm:max-w-[560px]">
+ <DialogHeader>
+ <div className="flex items-center justify-between gap-2">
+ <DialogTitle>
+ {title}
+ {!isSyncAll && connectorType ? (
+ <span className="text-muted-foreground font-normal">
+ {" "}
+ · {formatConnectorLabel(connectorType)}
+ </span>
+ ) : null}
+ </DialogTitle>
+ {hasOrphans ? (
+ <Badge
+ variant="secondary"
+ className="bg-accent-amber-foreground/15 text-accent-amber-foreground"
+ >
+ {totalOrphans} will be deleted
+ </Badge>
+ ) : null}
+ </div>
+ <DialogDescription className="pt-2 text-muted-foreground">
+ {description}
+ </DialogDescription>
+ </DialogHeader>
+
+ {hasOrphans ? (
+ <div className="rounded-md border border-accent-amber-foreground/30 bg-accent-amber-foreground/5 p-3">
+ <div className="flex items-center gap-2 mb-2 text-sm font-medium text-accent-amber-foreground">
+ <AlertTriangle className="h-4 w-4" />
+ Files to be deleted
+ </div>
+ <ScrollArea className="max-h-60">
+ {isSyncAll ? (
+ <div className="space-y-3 pr-2">
+ {Object.entries(orphansByType ?? {})
+ .filter(([, list]) => list.length > 0)
+ .map(([type, list], index, arr) => (
+ <div key={type}>
+ <div className="text-xs font-semibold uppercase tracking-wide text-muted-foreground mb-1">
+ {formatConnectorLabel(type)} ({list.length})
+ </div>
+ {renderOrphanList(list)}
+ {index < arr.length - 1 ? (
+ <Separator className="mt-3" />
+ ) : null}
+ </div>
+ ))}
+ </div>
+ ) : (
+ <div className="pr-2">{renderOrphanList(orphans ?? [])}</div>
+ )}
+ </ScrollArea>
+ </div>
+ ) : null}
+
+ {/* Surface "couldn't determine deletions" when strict gating aborted. */}
+ {isSyncAll && orphansAvailableByType
+ ? Object.entries(orphansAvailableByType)
+ .filter(([, available]) => !available)
+ .map(([type]) => (
+ <div
+ key={type}
+ className="text-xs text-muted-foreground italic"
+ >
+ Couldn't determine deletions for {formatConnectorLabel(type)}{" "}
+ (a connection may need re-authentication).
+ </div>
+ ))
+ : null}
+
+ <DialogFooter className="flex-row gap-2 justify-end">
+ <Button
+ type="button"
+ variant="ghost"
+ onClick={() => onOpenChange(false)}
+ disabled={busy}
+ size="sm"
+ >
+ Cancel
+ </Button>
+ <Button
+ type="button"
+ variant="default"
+ size="sm"
+ onClick={handleConfirm}
+ disabled={busy}
+ className={
+ hasOrphans
+ ? "flex items-center gap-2 !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"
+ : "flex items-center gap-2"
+ }
+ >
+ <RefreshCw className="h-3.5 w-3.5" />
+ {hasOrphans ? "Delete & sync" : "Confirm sync"}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ );
+};
diff --git a/src/api/connectors.py b/src/api/connectors.py
index e4213d828..3325a5c89 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1,20 +1,20 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
from fastapi import Depends, Request
-from pydantic import BaseModel
from fastapi.responses import JSONResponse, PlainTextResponse
-from connectors.sharepoint.utils import is_valid_sharepoint_url
+from pydantic import BaseModel
+
from config.settings import get_index_name
-from utils.logging_config import get_logger
-from utils.telemetry import TelemetryClient, Category, MessageId
+from connectors.sharepoint.utils import is_valid_sharepoint_url
from dependencies import (
get_connector_service,
get_current_user,
- get_rbac_service,
get_session_manager,
require_permission,
)
from session_manager import User
+from utils.logging_config import get_logger
+from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
@@ -77,16 +77,196 @@ async def get_synced_file_ids_for_connector(
return [], []
+async def get_synced_id_to_filename_map(
+ connector_type: str,
+ user_id: str,
+ session_manager,
+ jwt_token: str | None = None,
+) -> dict[str, str]:
+ """Return a {document_id: filename} map for files ingested under this connector_type.
+
+ Uses a sub-aggregation so each document_id is paired with its top filename in
+ a single OpenSearch round trip.
+ """
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": connector_type}},
+ "aggs": {
+ "by_document_id": {
+ "terms": {"field": "document_id", "size": 10000},
+ "aggs": {
+ "top_filename": {"terms": {"field": "filename", "size": 1}},
+ },
+ }
+ },
+ }
+
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ buckets = result.get("aggregations", {}).get("by_document_id", {}).get("buckets", [])
+
+ mapping: dict[str, str] = {}
+ for bucket in buckets:
+ doc_id = bucket.get("key")
+ if not doc_id:
+ continue
+ fn_buckets = bucket.get("top_filename", {}).get("buckets", [])
+ mapping[doc_id] = fn_buckets[0]["key"] if fn_buckets else ""
+ return mapping
+ except Exception as e:
+ logger.error(
+ "Failed to build id→filename map",
+ connector_type=connector_type,
+ error=str(e),
+ )
+ return {}
+
+
+async def compute_orphans_for_connector_type(
+ connector_type: str,
+ user_id: str,
+ connector_service,
+ session_manager,
+ jwt_token: str | None,
+ existing_file_ids: list[str],
+ id_to_filename: dict[str, str] | None = None,
+) -> list[dict[str, str]] | None:
+ """Compute orphan documents (ingested but no longer present at the source)
+ for this connector_type without deleting them.
+
+ Returns a list of {"document_id", "filename"} dicts. Returns None when strict
+ gating aborts the pass (unauthenticated connection or listing exception) so
+ callers can distinguish "no orphans" from "could not determine safely".
+ """
+ if not existing_file_ids:
+ return []
+
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user_id, connector_type=connector_type
+ )
+ active = [c for c in connections if c.is_active]
+ if not active:
+ logger.info(
+ "Skipping orphan compute — no active connections",
+ connector_type=connector_type,
+ )
+ return None
+
+ remote_ids: set = set()
+ for conn in active:
+ try:
+ connector = await connector_service.get_connector(conn.connection_id)
+ if not connector or not connector.is_authenticated:
+ logger.info(
+ "Skipping orphan compute — connection unauthenticated",
+ connector_type=connector_type,
+ connection_id=conn.connection_id,
+ )
+ return None
+ page_token = None
+ while True:
+ page = await connector.list_files(page_token=page_token)
+ for f in page.get("files", []):
+ fid = f.get("id")
+ if fid:
+ remote_ids.add(fid)
+ page_token = page.get("nextPageToken") or page.get("next_page_token")
+ if not page_token:
+ break
+ except Exception as e:
+ logger.warning(
+ "Skipping orphan compute — listing failed",
+ connector_type=connector_type,
+ connection_id=conn.connection_id,
+ error=str(e),
+ )
+ return None
+
+ orphan_ids = [fid for fid in existing_file_ids if fid not in remote_ids]
+ if not orphan_ids:
+ return []
+
+ fn_map = id_to_filename or {}
+ return [{"document_id": fid, "filename": fn_map.get(fid, "")} for fid in orphan_ids]
+
+
+async def delete_orphan_documents(
+ orphan_ids: list[str],
+ user_id: str,
+ session_manager,
+ jwt_token: str | None,
+) -> int:
+ """Delete OpenSearch chunks for the given orphan document IDs. Returns the
+ number of chunks deleted (0 on failure)."""
+ if not orphan_ids:
+ return 0
+ from .documents import delete_chunks_by_document_ids
+
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+ return await delete_chunks_by_document_ids(orphan_ids, opensearch_client, get_index_name())
+ except Exception as e:
+ logger.error(
+ "Orphan delete failed",
+ orphan_count=len(orphan_ids),
+ error=str(e),
+ )
+ return 0
+
+
+async def reconcile_orphans_for_connector_type(
+ connector_type: str,
+ user_id: str,
+ connector_service,
+ session_manager,
+ jwt_token: str | None,
+ existing_file_ids: list[str],
+) -> list[str]:
+ """Compute and delete orphans for a connector type. Thin wrapper around
+ compute_orphans_for_connector_type + delete_orphan_documents preserved for
+ callers that perform sync immediately after reconcile.
+
+ Returns the list of orphan file IDs that were deleted (or []).
+ """
+ orphans = await compute_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ )
+ if not orphans:
+ return []
+
+ orphan_ids = [o["document_id"] for o in orphans]
+ deleted = await delete_orphan_documents(
+ orphan_ids=orphan_ids,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+ logger.info(
+ "Orphan reconcile complete",
+ connector_type=connector_type,
+ orphan_count=len(orphan_ids),
+ deleted_chunks=deleted,
+ )
+ return orphan_ids
+
+
class ConnectorSyncBody(BaseModel):
- max_files: Optional[int] = None
- selected_files: Optional[List[Any]] = None
+ max_files: int | None = None
+ selected_files: list[Any] | None = None
# When True, ingest ALL files from the connector (bypasses the existing-files gate).
# Used by direct-sync providers like IBM COS on initial ingest.
sync_all: bool = False
# When set, only ingest files from these buckets (IBM COS specific).
- bucket_filter: Optional[List[str]] = None
+ bucket_filter: list[str] | None = None
# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).
- settings: Optional[Dict[str, Any]] = None
+ settings: dict[str, Any] | None = None
async def list_connectors(
@@ -279,6 +459,18 @@ async def connector_sync(
connector_type=connector_type,
file_count=len(existing_file_ids),
)
+ # Reconcile orphans (files deleted at the source) before re-syncing.
+ # Strict gating: skip when sync is capped — we'd see a partial remote
+ # listing and delete legitimate files.
+ if body.max_files is None:
+ await reconcile_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ )
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
@@ -358,7 +550,7 @@ async def connector_status(
)
else:
logger.debug(
- f"connector_status: Connector has no base_url or sharepoint_url attribute"
+ "connector_status: Connector has no base_url or sharepoint_url attribute"
)
connection_details[connection.connection_id] = {
@@ -739,6 +931,17 @@ async def sync_all_connectors(
connector_type=connector_type,
file_count=len(existing_file_ids),
)
+ # Reconcile orphans (files deleted at the source) before re-syncing.
+ # sync_all_connectors has no caps or filters, so gating reduces
+ # to the strict checks inside the helper.
+ await reconcile_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ )
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
@@ -817,6 +1020,137 @@ async def sync_all_connectors(
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
+CLOUD_CONNECTOR_TYPES = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]
+
+
+async def _preview_orphans_for_connector_type(
+ connector_type: str,
+ user_id: str,
+ connector_service,
+ session_manager,
+ jwt_token: str | None,
+) -> tuple[list[dict[str, str]] | None, int]:
+ """Helper: compute orphans (no deletion) + return total synced count.
+
+ Returns (orphans, synced_count). `orphans` is None when strict gating aborts
+ (so the caller can surface a "couldn't determine" state); [] when no orphans.
+ """
+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+
+ synced_count = len(existing_file_ids) if existing_file_ids else len(existing_filenames)
+ if not existing_file_ids:
+ # No document_ids to diff against (e.g. Langflow-only ingest). Filename-only
+ # fallback can't detect orphans safely — surface empty list.
+ return [], synced_count
+
+ id_to_filename = await get_synced_id_to_filename_map(
+ connector_type=connector_type,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+
+ orphans = await compute_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ id_to_filename=id_to_filename,
+ )
+ return orphans, synced_count
+
+
+async def connector_sync_preview(
+ connector_type: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(require_permission("connectors:use")),
+):
+ """Preview the impact of syncing a connector type without performing any
+ deletion or ingest. Returns the list of orphan files (present in OpenSearch
+ but no longer at the source) by filename, plus the total synced count.
+ """
+ try:
+ orphans, synced_count = await _preview_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=user.jwt_token,
+ )
+ return JSONResponse(
+ {
+ "connector_type": connector_type,
+ "synced_count": synced_count,
+ "orphans": orphans or [],
+ "orphans_available": orphans is not None,
+ },
+ status_code=200,
+ )
+ except Exception as e:
+ logger.error("Sync preview failed", connector_type=connector_type, error=str(e))
+ return JSONResponse({"error": f"Sync preview failed: {str(e)}"}, status_code=500)
+
+
+async def connectors_sync_all_preview(
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(require_permission("connectors:use")),
+):
+ """Preview the impact of sync-all-connectors across every cloud connector
+ type. Returns orphan filenames grouped by connector_type plus a per-type
+ synced count.
+ """
+ try:
+ orphans_by_type: dict[str, list[dict[str, str]]] = {}
+ synced_count_by_type: dict[str, int] = {}
+ orphans_available_by_type: dict[str, bool] = {}
+
+ for connector_type in CLOUD_CONNECTOR_TYPES:
+ try:
+ orphans, synced_count = await _preview_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=user.jwt_token,
+ )
+ except Exception as e:
+ logger.warning(
+ "Sync-all preview: per-connector failure",
+ connector_type=connector_type,
+ error=str(e),
+ )
+ orphans, synced_count = None, 0
+
+ # Only include connector types that have something synced.
+ if synced_count == 0 and not orphans:
+ continue
+
+ synced_count_by_type[connector_type] = synced_count
+ orphans_by_type[connector_type] = orphans or []
+ orphans_available_by_type[connector_type] = orphans is not None
+
+ return JSONResponse(
+ {
+ "orphans_by_type": orphans_by_type,
+ "synced_count_by_type": synced_count_by_type,
+ "orphans_available_by_type": orphans_available_by_type,
+ },
+ status_code=200,
+ )
+ except Exception as e:
+ logger.error("Sync-all preview failed", error=str(e))
+ return JSONResponse({"error": f"Sync-all preview failed: {str(e)}"}, status_code=500)
+
+
async def connector_token(
connector_type: str,
connection_id: str,
@@ -942,9 +1276,9 @@ async def browse_connection_files(
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
user: User = Depends(get_current_user),
- bucket: Optional[str] = None,
- search: Optional[str] = None,
- page_token: Optional[str] = None,
+ bucket: str | None = None,
+ search: str | None = None,
+ page_token: str | None = None,
max_files: int = 100,
):
"""
diff --git a/src/api/documents.py b/src/api/documents.py
index ce9b6275b..421f966fe 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -1,10 +1,10 @@
from fastapi import Depends
-from pydantic import BaseModel
from fastapi.responses import JSONResponse
-from utils.logging_config import get_logger
+from pydantic import BaseModel
-from dependencies import get_session_manager, get_current_user, require_permission
+from dependencies import get_current_user, get_session_manager, require_permission
from session_manager import User
+from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -122,10 +122,24 @@ async def delete_documents_by_filename_core(
)
+async def delete_chunks_by_document_ids(
+ document_ids: list[str],
+ opensearch_client,
+ index_name: str,
+) -> int:
+ """Bulk delete OpenSearch chunks by document_id. Returns deleted count."""
+ if not document_ids:
+ return 0
+ body = {"query": {"terms": {"document_id": document_ids}}}
+ res = await opensearch_client.delete_by_query(index=index_name, body=body, conflicts="proceed")
+ return res.get("deleted", 0)
+
+
async def _ensure_index_exists(jwt_token: str = None):
"""Create the OpenSearch index if it doesn't exist yet."""
+ from config.settings import IBM_AUTH_ENABLED
+ from config.settings import clients as app_clients
from main import init_index
- from config.settings import IBM_AUTH_ENABLED, clients as app_clients
opensearch_client = None
if IBM_AUTH_ENABLED and jwt_token:
@@ -147,8 +161,8 @@ async def check_filename_exists(
try:
opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)
- from utils.opensearch_queries import build_filename_search_body
from utils.file_utils import get_filename_aliases
+ from utils.opensearch_queries import build_filename_search_body
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index 89072bd84..dd44b3d11 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -239,12 +239,24 @@ def register_internal_routes(app: FastAPI):
methods=["POST"],
tags=["internal"],
)
+ app.add_api_route(
+ "/connectors/{connector_type}/sync-preview",
+ connectors.connector_sync_preview,
+ methods=["POST"],
+ tags=["internal"],
+ )
app.add_api_route(
"/connectors/sync-all",
connectors.sync_all_connectors,
methods=["POST"],
tags=["internal"],
)
+ app.add_api_route(
+ "/connectors/sync-all-preview",
+ connectors.connectors_sync_all_preview,
+ methods=["POST"],
+ tags=["internal"],
+ )
app.add_api_route(
"/connectors/{connector_type}/status",
connectors.connector_status,
diff --git a/src/connectors/service.py b/src/connectors/service.py
index fe463f590..146c70116 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -1,11 +1,10 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
+from utils.file_utils import clean_connector_filename, get_file_extension
from utils.logging_config import get_logger
from .base import BaseConnector, ConnectorDocument
from .connection_manager import ConnectionManager
-from utils.file_utils import get_file_extension, clean_connector_filename
-
logger = get_logger(__name__)
@@ -38,7 +37,7 @@ class ConnectorService:
"""Initialize the service by loading existing connections"""
await self.connection_manager.load_connections()
- async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:
+ async def get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID"""
return await self.connection_manager.get_connector(connection_id)
@@ -50,7 +49,7 @@ class ConnectorService:
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Process a document from a connector using existing processing pipeline"""
# Create temporary file from document content
@@ -153,6 +152,9 @@ class ConnectorService:
"source": """
ctx._source.source_url = params.source_url;
ctx._source.connector_type = params.connector_type;
+ if (params.filename != null) {
+ ctx._source.filename = params.filename;
+ }
if (params.created_time != null) {
ctx._source.created_time = params.created_time;
}
@@ -166,6 +168,7 @@ class ConnectorService:
"params": {
"source_url": document.source_url,
"connector_type": connector_type,
+ "filename": document.filename,
"created_time": document.created_time.isoformat()
if document.created_time
else None,
@@ -227,7 +230,7 @@ class ConnectorService:
raise ValueError(f"Connection '{connection_id}' not authenticated")
# Collect files to process (limited by max_files)
- files_to_process = []
+ files_to_process: list[dict[str, Any]] = []
page_token = None
# Calculate page size to minimize API calls
@@ -236,7 +239,7 @@ class ConnectorService:
while True:
# List files from connector with limit
logger.debug("Calling list_files", page_size=page_size, page_token=page_token)
- file_list = await connector.list_files(page_token, limit=page_size)
+ file_list = await connector.list_files(page_token, max_files=page_size)
logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
files = file_list["files"]
@@ -311,10 +314,10 @@ class ConnectorService:
self,
connection_id: str,
user_id: str,
- file_ids: List[str],
+ file_ids: list[str],
jwt_token: str = None,
- file_infos: List[Dict[str, Any]] = None,
- ingest_settings: Optional[Dict[str, Any]] = None,
+ file_infos: list[dict[str, Any]] = None,
+ ingest_settings: dict[str, Any] | None = None,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
@@ -370,8 +373,8 @@ class ConnectorService:
try:
# Set the file_ids we want to sync in the connector's config
if hasattr(connector, "cfg"):
- connector.cfg.file_ids = file_ids # type: ignore
- connector.cfg.folder_ids = None # type: ignore
+ connector.cfg.file_ids = file_ids
+ connector.cfg.folder_ids = None
# Get the expanded list of file IDs (folders will be expanded to their contents)
# This uses the connector's list_files() which calls _iter_selected_items()
@@ -416,8 +419,8 @@ class ConnectorService:
finally:
# Restore original config values
if hasattr(connector, "cfg"):
- connector.cfg.file_ids = original_file_ids # type: ignore
- connector.cfg.folder_ids = original_folder_ids # type: ignore
+ connector.cfg.file_ids = original_file_ids
+ connector.cfg.folder_ids = original_folder_ids
# Create custom processor for specific connector files
from models.processors import ConnectorFileProcessor
@@ -456,6 +459,6 @@ class ConnectorService:
return task_id
- async def _get_connector(self, connection_id: str) -> Optional[BaseConnector]:
+ async def _get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID (alias for get_connector)"""
return await self.get_connector(connection_id)
diff --git a/src/models/processors.py b/src/models/processors.py
index d9e225d2b..01df83a88 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1,13 +1,28 @@
+import asyncio
+import datetime
+import mimetypes
+import os
+import time
from typing import TYPE_CHECKING, Any
+from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
+from utils.document_processing import (
+ extract_relevant,
+ process_text_file,
+ resplit_chunks_character_windows,
+)
+from utils.embedding_fields import ensure_embedding_field_exists
from utils.file_utils import (
+ auto_cleanup_tempfile,
clean_connector_filename,
get_file_extension,
get_filename_aliases,
)
+from utils.hash_utils import hash_id
from utils.logging_config import get_logger
+from utils.opensearch_queries import build_filename_delete_body, build_filename_search_body
-from .tasks import FileTask, UploadTask
+from .tasks import FileTask, TaskStatus, UploadTask
logger = get_logger(__name__)
@@ -32,10 +47,6 @@ class TaskProcessor:
Check if a document with the given hash already exists in OpenSearch.
Consolidated hash checking for all processors.
"""
- import asyncio
-
- from config.settings import get_index_name
-
max_retries = 3
retry_delay = 1.0
@@ -78,11 +89,6 @@ class TaskProcessor:
Check if a document with the given filename already exists in OpenSearch.
Returns True if any chunks with this filename exist.
"""
- import asyncio
-
- from config.settings import get_index_name
- from utils.opensearch_queries import build_filename_search_body
-
max_retries = 3
retry_delay = 1.0
@@ -149,9 +155,6 @@ class TaskProcessor:
"""
Delete all chunks of a document with the given filename from OpenSearch.
"""
- from config.settings import get_index_name
- from utils.opensearch_queries import build_filename_delete_body
-
try:
deleted_count = 0
candidate_filenames = get_filename_aliases(filename)
@@ -204,20 +207,7 @@ class TaskProcessor:
chunk_overlap: Overlap between windows; must be less than ``chunk_size``.
acl: DocumentACL instance with access control information
"""
- import datetime
-
- from config.settings import (
- clients,
- get_embedding_model,
- get_index_name,
- get_openrag_config,
- )
from services.document_service import chunk_texts_for_embeddings
- from utils.document_processing import (
- extract_relevant,
- resplit_chunks_character_windows,
- )
- from utils.embedding_fields import ensure_embedding_field_exists
# Use provided embedding model or configured model.
# get_embedding_model() returns empty string when Langflow ingest is enabled,
@@ -241,14 +231,10 @@ class TaskProcessor:
)
# Check if this is a .txt or .md file - use simple processing instead of docling
- import os
-
file_ext = os.path.splitext(file_path)[1].lower()
if file_ext in (".txt", ".md"):
# Simple text file processing without docling
- from utils.document_processing import process_text_file
-
logger.info(
"Processing as plain text file (bypassing docling)",
file_path=file_path,
@@ -418,12 +404,6 @@ class DocumentFileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a regular file path using consolidated methods"""
- import os
- import time
-
- from models.tasks import TaskStatus
- from utils.hash_utils import hash_id
-
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -499,11 +479,6 @@ class ConnectorFileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using consolidated methods"""
- import time
-
- from models.tasks import TaskStatus
- from utils.hash_utils import hash_id
-
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -519,7 +494,23 @@ class ConnectorFileProcessor(TaskProcessor):
raise ValueError(f"Connection '{self.connection_id}' not found")
# Get file content from connector
- document = await connector.get_file_content(file_id)
+ try:
+ document = await connector.get_file_content(file_id)
+ except (FileNotFoundError, ValueError) as e:
+ msg = str(e).lower()
+ if "not found" in msg or "404" in msg:
+ logger.warning(
+ "File no longer exists at source — skipping",
+ file_id=file_id,
+ connection_id=self.connection_id,
+ error=str(e),
+ )
+ file_task.status = TaskStatus.SKIPPED
+ file_task.result = {"status": "skipped", "reason": "deleted_at_source"}
+ file_task.updated_at = time.time()
+ upload_task.successful_files += 1
+ return
+ raise
# Update filename in task once we have it from the connector
file_task.filename = clean_connector_filename(document.filename, document.mimetype)
@@ -528,8 +519,6 @@ class ConnectorFileProcessor(TaskProcessor):
raise ValueError("user_id not provided to ConnectorFileProcessor")
# Create temporary file from document content
- from utils.file_utils import auto_cleanup_tempfile
-
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
@@ -626,11 +615,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a connector file using LangflowConnectorService"""
- import time
-
- from models.tasks import TaskStatus
- from utils.hash_utils import hash_id
-
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -655,8 +639,6 @@ class LangflowConnectorFileProcessor(TaskProcessor):
raise ValueError("user_id not provided to LangflowConnectorFileProcessor")
# Create temporary file and compute hash to check for duplicates
- from utils.file_utils import auto_cleanup_tempfile
-
suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
@@ -734,19 +716,9 @@ class S3FileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
- import time
- import asyncio
- import datetime
- from config.settings import clients, get_embedding_model, get_index_name
-
- from models.tasks import TaskStatus
-
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
- from utils.file_utils import auto_cleanup_tempfile
- from utils.hash_utils import hash_id
-
try:
with auto_cleanup_tempfile() as tmp_path:
# Download object to temporary file
@@ -827,12 +799,6 @@ class LangflowFileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a file path using LangflowFileService upload_and_ingest_file"""
- import mimetypes
- import os
- import time
-
- from models.tasks import TaskStatus
-
# Update task status
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
diff --git a/src/models/tasks.py b/src/models/tasks.py
index 40bda40ff..747579f62 100644
--- a/src/models/tasks.py
+++ b/src/models/tasks.py
@@ -2,7 +2,7 @@ import itertools
import time
from dataclasses import dataclass, field
from enum import Enum
-from typing import ClassVar, Dict, Optional
+from typing import ClassVar
class TaskStatus(Enum):
@@ -10,6 +10,7 @@ class TaskStatus(Enum):
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
+ SKIPPED = "skipped"
class DoclingPhaseStatus(Enum):
@@ -40,17 +41,17 @@ class IngestionPhase(Enum):
class FileTask:
file_path: str
status: TaskStatus = TaskStatus.PENDING
- result: Optional[dict] = None
- error: Optional[str] = None
+ result: dict | None = None
+ error: str | None = None
retry_count: int = 0
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
- filename: Optional[str] = None # Original filename for display
# Two-phase ingestion fields. Only meaningful for processors that submit
# files to Docling Serve and then trigger Langflow (i.e. LangflowFileProcessor).
- docling_task_id: Optional[str] = None
+ docling_task_id: str | None = None
docling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDING
phase: IngestionPhase = IngestionPhase.DOCLING
+ filename: str | None = None # Original filename for display
@property
def duration_seconds(self) -> float:
@@ -67,7 +68,7 @@ class UploadTask:
processed_files: int = 0
successful_files: int = 0
failed_files: int = 0
- file_tasks: Dict[str, FileTask] = field(default_factory=dict)
+ file_tasks: dict[str, FileTask] = field(default_factory=dict)
status: TaskStatus = TaskStatus.PENDING
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.py
new file mode 100644
index 000000000..005a1477d
--- /dev/null
+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py
@@ -0,0 +1,62 @@
+"""Unit tests for the bulk-delete helper used by the orphan-reconcile pass.
+
+Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:
+- empty input is a no-op (no OpenSearch call)
+- non-empty input issues a single delete_by_query with terms(document_id, ...)
+- returns the count reported by OpenSearch
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+async def test_empty_ids_short_circuits_without_calling_opensearch():
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = AsyncMock()
+ deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_issues_single_delete_by_query_with_terms_filter():
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = AsyncMock()
+ opensearch_client.delete_by_query.return_value = {"deleted": 12}
+
+ ids = ["abc", "def", "ghi"]
+ deleted = await delete_chunks_by_document_ids(ids, opensearch_client, "test-index")
+
+ assert deleted == 12
+ opensearch_client.delete_by_query.assert_awaited_once()
+ call = opensearch_client.delete_by_query.await_args
+ assert call.kwargs["index"] == "test-index"
+ assert call.kwargs["body"] == {"query": {"terms": {"document_id": ids}}}
+ # conflicts="proceed" so a concurrent reindex doesn't abort the bulk delete.
+ assert call.kwargs.get("conflicts") == "proceed"
+
+
+async def test_returns_zero_when_response_missing_deleted_field():
+ """Defensive: if OpenSearch returns an unexpected payload shape, we
+ should not crash — we surface 0 deletions."""
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = AsyncMock()
+ opensearch_client.delete_by_query.return_value = {} # no "deleted" key
+
+ deleted = await delete_chunks_by_document_ids(["abc"], opensearch_client, "test-index")
+
+ assert deleted == 0
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
new file mode 100644
index 000000000..bd9a1a187
--- /dev/null
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -0,0 +1,338 @@
+"""Unit tests for `reconcile_orphans_for_connector_type` in `src/api/connectors.py`.
+
+This is the orphan-deletion safety net for the connector sync flow. The
+function must:
+- Compute orphans = indexed_ids - union(remote_ids across all active
+ connections of this connector_type for this user).
+- Apply STRICT gating: any unauthenticated connection or listing
+ exception aborts the pass with 0 deletes (false-negative > false-positive).
+- Preserve files that exist in any active connection of the type
+ (multi-connection isolation).
+"""
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_connection(connection_id: str, is_active: bool = True):
+ """Lightweight stand-in for ConnectionConfig — only the attributes
+ reconcile_orphans_for_connector_type reads."""
+ return SimpleNamespace(connection_id=connection_id, is_active=is_active)
+
+
+def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):
+ """Build an AsyncMock connector that reports the given file IDs from
+ list_files() (single page, no pagination)."""
+ connector = MagicMock()
+ connector.is_authenticated = authenticated
+ if raise_on_list:
+ connector.list_files = AsyncMock(side_effect=RuntimeError("graph 503"))
+ else:
+ connector.list_files = AsyncMock(
+ return_value={"files": [{"id": fid} for fid in remote_file_ids]}
+ )
+ return connector
+
+
+def _make_service(connections, connector_lookup):
+ """Build a connector_service stub.
+
+ `connections` is the list returned by list_connections.
+ `connector_lookup` maps connection_id -> connector mock (or None).
+ """
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=connections)
+
+ async def _get_connector(connection_id):
+ return connector_lookup.get(connection_id)
+
+ service.get_connector = AsyncMock(side_effect=_get_connector)
+ return service
+
+
+def _make_session_manager(opensearch_client):
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ return sm
+
+
+async def test_empty_existing_file_ids_returns_zero_without_calls():
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock()
+ sm = _make_session_manager(AsyncMock())
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=[],
+ )
+
+ assert deleted == 0
+ service.connection_manager.list_connections.assert_not_awaited()
+
+
+async def test_no_active_connections_skips_reconcile():
+ """If every connection is inactive, we have no remote view — skip."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ inactive = [_make_connection("c1", is_active=False)]
+ service = _make_service(inactive, connector_lookup={})
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_unauthenticated_connection_aborts_pass(monkeypatch):
+ """STRICT GATING: even one unauthenticated connector aborts the pass.
+ Otherwise we'd wrongly mark a file as an orphan because we couldn't
+ list the connection that holds it."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=[], authenticated=False)
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
+ connector.list_files.assert_not_awaited()
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_listing_exception_aborts_pass():
+ """STRICT GATING: a transient list_files error aborts the pass.
+ Better to leave orphans for one cycle than delete legitimate files."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=[], raise_on_list=True)
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_happy_path_deletes_orphans():
+ """Indexed has [a, b, c]; remote has [a, c] -> orphan = [b]."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a", "c"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = AsyncMock()
+ opensearch_client.delete_by_query.return_value = {"deleted": 7}
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b", "c"],
+ )
+
+ assert deleted == 7
+ opensearch_client.delete_by_query.assert_awaited_once()
+ body = opensearch_client.delete_by_query.await_args.kwargs["body"]
+ assert body == {"query": {"terms": {"document_id": ["b"]}}}
+
+
+async def test_no_orphans_skips_delete_call():
+ """If every indexed ID still exists remotely, we must not issue an
+ empty delete_by_query."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a", "b"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_multi_connection_union_preserves_files_present_in_any_connection():
+ """Multi-connection isolation:
+ - User has conn-A and conn-B (both SharePoint, different sites).
+ - Indexed has [a, b]. conn-A has [a]. conn-B has [b].
+ - Neither file is an orphan — both should be preserved."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn_a = _make_connection("conn-a")
+ conn_b = _make_connection("conn-b")
+ connector_a = _make_connector(remote_file_ids=["a"])
+ connector_b = _make_connector(remote_file_ids=["b"])
+ service = _make_service(
+ [conn_a, conn_b],
+ connector_lookup={"conn-a": connector_a, "conn-b": connector_b},
+ )
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
+ """If conn-A lists fine but conn-B is unauthenticated, we MUST NOT
+ treat files only in conn-B as orphans. Strict gating bails out."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn_a = _make_connection("conn-a")
+ conn_b = _make_connection("conn-b")
+ connector_a = _make_connector(remote_file_ids=["a"])
+ connector_b = _make_connector(remote_file_ids=[], authenticated=False)
+ service = _make_service(
+ [conn_a, conn_b],
+ connector_lookup={"conn-a": connector_a, "conn-b": connector_b},
+ )
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"], # b would look like an orphan if we trusted conn-A alone
+ )
+
+ assert deleted == 0
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_paginated_listing_aggregates_all_pages():
+ """Remote listings are paginated. The reconcile must walk every page
+ before computing the orphan set, otherwise files on page 2 look like
+ orphans."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = MagicMock()
+ connector.is_authenticated = True
+ pages = [
+ {"files": [{"id": "a"}], "nextPageToken": "tok-1"},
+ {"files": [{"id": "b"}, {"id": "c"}]}, # last page, no token
+ ]
+ connector.list_files = AsyncMock(side_effect=pages)
+
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = AsyncMock()
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b", "c"],
+ )
+
+ assert deleted == 0
+ assert connector.list_files.await_count == 2
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+
+async def test_delete_failure_does_not_raise():
+ """If the bulk delete itself blows up, the helper must swallow it and
+ return 0. The caller (sync) should still proceed to re-sync surviving
+ files — leaving orphans is recoverable, raising is not."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+
+ opensearch_client = AsyncMock()
+ opensearch_client.delete_by_query.side_effect = RuntimeError("opensearch unavailable")
+ sm = _make_session_manager(opensearch_client)
+
+ deleted = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert deleted == 0
diff --git a/tests/unit/connectors/__init__.py b/tests/unit/connectors/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.py
new file mode 100644
index 000000000..14cbccaa2
--- /dev/null
+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py
@@ -0,0 +1,134 @@
+"""Connector rename fix: _update_connector_metadata must rewrite the indexed
+`filename` field so renamed source files don't leave their old name in
+OpenSearch chunks.
+
+Pins the painless script extension in src/connectors/service.py
+`_update_connector_metadata`.
+"""
+
+import sys
+from datetime import datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_service():
+ """Build a ConnectorService whose only wired-up surface is what
+ _update_connector_metadata touches: an opensearch client (via
+ session_manager) and an index_name."""
+ from connectors.service import ConnectorService
+
+ service = ConnectorService.__new__(ConnectorService)
+ service.session_manager = MagicMock()
+ service.index_name = "test-index"
+
+ opensearch_client = AsyncMock()
+ service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ return service, opensearch_client
+
+
+def _make_document(filename: str = "renamed.pdf"):
+ from connectors.base import ConnectorDocument, DocumentACL
+
+ return ConnectorDocument(
+ id="graph-item-id-stable",
+ filename=filename,
+ mimetype="application/pdf",
+ content=b"",
+ source_url="https://contoso.sharepoint.com/.../renamed.pdf",
+ acl=DocumentACL(owner="alice"),
+ modified_time=datetime(2026, 5, 7),
+ created_time=datetime(2026, 5, 1),
+ metadata={"site": "marketing"},
+ )
+
+
+async def test_update_includes_filename_in_script_and_params(monkeypatch):
+ """The painless script must reference params.filename, and params must
+ carry document.filename as-is."""
+ service, opensearch_client = _make_service()
+
+ # update_document_acl is invoked first; stub it to a no-op so the test
+ # focuses on the metadata update_by_query.
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ document = _make_document(filename="Report-FY26.docx")
+ await service._update_connector_metadata(
+ document, owner_user_id="alice", connector_type="sharepoint"
+ )
+
+ opensearch_client.update_by_query.assert_awaited_once()
+ call = opensearch_client.update_by_query.await_args
+ body = call.kwargs["body"]
+ script = body["script"]
+ params = script["params"]
+
+ assert "params.filename" in script["source"], (
+ "painless script must read params.filename so renamed files get "
+ "the new name written through to indexed chunks"
+ )
+ assert params["filename"] == "Report-FY26.docx"
+ # Sanity: scope is by document_id (the stable connector ID).
+ assert body["query"] == {"term": {"document_id": "graph-item-id-stable"}}
+
+
+async def test_filename_passed_raw_not_cleaned(monkeypatch):
+ """process_document_standard indexes chunks with the raw document.filename
+ (see src/connectors/service.py:80 + src/models/processors.py:323).
+ The metadata update must use the SAME raw value or chunks drift.
+ """
+ service, opensearch_client = _make_service()
+
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ # A name with a quirky extension/suffix that clean_connector_filename
+ # would normalize. We pass it through verbatim.
+ raw_name = "Q4 Report (final).docx"
+ document = _make_document(filename=raw_name)
+
+ await service._update_connector_metadata(
+ document, owner_user_id="alice", connector_type="sharepoint"
+ )
+
+ params = opensearch_client.update_by_query.await_args.kwargs["body"]["script"]["params"]
+ assert params["filename"] == raw_name
+
+
+async def test_filename_overwritten_for_existing_indexed_chunks(monkeypatch):
+ """End-to-end intent: when SharePoint rename triggers the unchanged ->
+ metadata-update path, the painless update writes the new filename.
+ Verified via the captured update_by_query body."""
+ service, opensearch_client = _make_service()
+
+ async def _noop_acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _noop_acl)
+
+ # Simulate the post-rename state: same ID, new filename.
+ document = _make_document(filename="renamed-after-edit.pdf")
+ await service._update_connector_metadata(
+ document, owner_user_id="alice", connector_type="sharepoint"
+ )
+
+ body = opensearch_client.update_by_query.await_args.kwargs["body"]
+ # The script must carry the new filename in its params, AND the source
+ # must assign it onto _source.filename for every matching chunk.
+ assert body["script"]["params"]["filename"] == "renamed-after-edit.pdf"
+ assert "ctx._source.filename = params.filename" in body["script"]["source"]