OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: support file level ingestion and files listing search (#1536)
Rico Furtadoe74483c04069
diff --git a/frontend/app/api/queries/useBrowseConnectionFiles.ts b/frontend/app/api/queries/useBrowseConnectionFiles.tsnew file mode 100644index 000000000..fa0c12d31--- /dev/null+++ b/frontend/app/api/queries/useBrowseConnectionFiles.ts@@ -0,0 +1,75 @@+import {+ type UseQueryOptions,+ useQuery,+ useQueryClient,+} from "@tanstack/react-query";++export interface RemoteFile {+ id: string;+ name: string;+ bucket: string;+ key: string;+ size: number;+ modified_time: string;+ is_ingested: boolean;+}++export interface BrowseConnectionFilesParams {+ connectorType: string;+ connectionId: string;+ bucket?: string;+ search?: string;+ pageToken?: string;+ maxFiles?: number;+}++export interface BrowseConnectionFilesResponse {+ files: RemoteFile[];+ next_page_token: string | null;+ total_remote: number;+ total_ingested: number;+}++export const useBrowseConnectionFiles = (+ params: BrowseConnectionFilesParams,+ options?: Omit<+ UseQueryOptions<BrowseConnectionFilesResponse>,+ "queryKey" | "queryFn"+ >,+) => {+ const queryClient = useQueryClient();++ async function fetchFiles(): Promise<BrowseConnectionFilesResponse> {+ const searchParams = new URLSearchParams();++ if (params.bucket) searchParams.set("bucket", params.bucket);+ if (params.search) searchParams.set("search", params.search);+ if (params.pageToken) searchParams.set("page_token", params.pageToken);+ if (params.maxFiles) searchParams.set("max_files", String(params.maxFiles));++ const url = `/api/connectors/${params.connectorType}/${params.connectionId}/browse?${searchParams.toString()}`;+ const response = await fetch(url);++ if (!response.ok) {+ const errorData = await response+ .json()+ .catch(() => ({ error: "Unknown error" }));+ throw new Error(+ errorData.error || `Failed to browse files: ${response.status}`,+ );+ }++ return response.json();+ }++ return useQuery(+ {+ queryKey: ["browseConnectionFiles", params],+ queryFn: fetchFiles,+ retry: false,+ enabled: Boolean(params.connectorType && params.connectionId),+ ...options,+ },+ queryClient,+ );+};diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.tsnew file mode 100644index 000000000..b5b06f385--- /dev/null+++ b/frontend/app/api/queries/useListFiles.ts@@ -0,0 +1,97 @@+import {+ type UseQueryOptions,+ useQuery,+ useQueryClient,+} from "@tanstack/react-query";+import type { File } from "./useGetSearchQuery";++export interface ListFilesParams {+ page?: number;+ pageSize?: number;+ sortBy?: string;+ sortOrder?: "asc" | "desc";+ connectorType?: string;+ mimetype?: string;+ owner?: string;+ search?: string;+}++export interface ListFilesResponse {+ files: File[];+ total: number;+ page: number;+ page_size: number;+}++export const useListFiles = (+ params: ListFilesParams = {},+ options?: Omit<UseQueryOptions<ListFilesResponse>, "queryKey" | "queryFn">,+) => {+ const queryClient = useQueryClient();++ async function fetchFiles(): Promise<ListFilesResponse> {+ const searchParams = new URLSearchParams();++ if (params.page) searchParams.set("page", String(params.page));+ if (params.pageSize) searchParams.set("page_size", String(params.pageSize));+ if (params.sortBy) searchParams.set("sort_by", params.sortBy);+ if (params.sortOrder) searchParams.set("sort_order", params.sortOrder);+ if (params.connectorType)+ searchParams.set("connector_type", params.connectorType);+ if (params.mimetype) searchParams.set("mimetype", params.mimetype);+ if (params.owner) searchParams.set("owner", params.owner);+ if (params.search) searchParams.set("search", params.search);++ const url = `/api/files?${searchParams.toString()}`;+ const response = await fetch(url);++ if (!response.ok) {+ const errorData = await response+ .json()+ .catch(() => ({ error: "Unknown error" }));+ throw new Error(+ errorData.error || `Failed to list files: ${response.status}`,+ );+ }++ const data = await response.json();++ // Map server response to File interface+ const files: File[] = (data.files || []).map(+ (f: Record<string, unknown>) => ({+ filename: (f.filename as string) || "",+ mimetype: (f.mimetype as string) || "",+ chunkCount: (f.chunk_count as number) || 0,+ source_url: (f.source_url as string) || "",+ owner: (f.owner as string) || "",+ owner_name: (f.owner_name as string) || "",+ owner_email: (f.owner_email as string) || "",+ size: (f.file_size as number) || 0,+ connector_type: (f.connector_type as string) || "local",+ embedding_model: f.embedding_model as string | undefined,+ embedding_dimensions: f.embedding_dimensions as number | undefined,+ allowed_users: (f.allowed_users as string[]) || [],+ allowed_groups: (f.allowed_groups as string[]) || [],+ status: "active" as const,+ }),+ );++ return {+ files,+ total: data.total || 0,+ page: data.page || 1,+ page_size: data.page_size || 25,+ };+ }++ return useQuery(+ {+ queryKey: ["listFiles", params],+ placeholderData: (prev: ListFilesResponse | undefined) => prev,+ queryFn: fetchFiles,+ retry: false,+ ...options,+ },+ queryClient,+ );+};diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex c7a4d0c79..6fa0a7eed 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -25,6 +25,7 @@ import {type SearchResult,useGetSearchQuery,} from "../api/queries/useGetSearchQuery";+import { useListFiles } from "../api/queries/useListFiles";import "@/components/AgGrid/registerAgGridModules";import "@/components/AgGrid/agGridStyles.css";import { toast } from "sonner";@@ -209,17 +210,52 @@ function SearchPage() {getFailedFileKey,]);+ // Use server-side file listing for default/wildcard view,+ // fall back to search-based aggregation for semantic queries+ const isWildcardQuery =+ !queryOverride ||+ queryOverride.trim() === "" ||+ queryOverride.trim() === "*";++ const {+ data: listFilesData,+ isLoading: isListFilesLoading,+ error: listFilesError,+ isError: isListFilesError,+ } = useListFiles(+ {+ pageSize: 100,+ search: isWildcardQuery ? undefined : queryOverride,+ connectorType: parsedFilterData?.filters?.connector_types?.[0],+ mimetype: parsedFilterData?.filters?.document_types?.[0],+ owner: parsedFilterData?.filters?.owners?.[0],+ },+ {+ refetchInterval: 5000,+ enabled: isWildcardQuery,+ },+ );+const {data: searchData = EMPTY_SEARCH_RESULT,isLoading: isSearchLoading,- error,- isError,+ error: searchError,+ isError: isSearchError,} = useGetSearchQuery(queryOverride, parsedFilterData, {refetchInterval: 5000,+ enabled: !isWildcardQuery,});const { files: searchFiles, warnings: searchWarnings } =searchData as SearchResult;+ // Merge data from whichever source is active+ const effectiveData: File[] = isWildcardQuery+ ? (listFilesData?.files ?? [])+ : searchFiles;+ const isLoading = isWildcardQuery ? isListFilesLoading : isSearchLoading;+ const error = isWildcardQuery ? listFilesError : searchError;+ const isError = isWildcardQuery ? isListFilesError : isSearchError;+const isOpenragDocsRow = useCallback((file?: File) => {return (file?.connector_type === "openrag_docs" ||@@ -312,7 +348,7 @@ function SearchPage() {}}, [isError, error]);const fileResults = buildKnowledgeTableRows(- searchFiles,+ effectiveData,taskFiles,!!parsedFilterData,);@@ -836,7 +872,7 @@ function SearchPage() {className="w-full overflow-auto border"columnDefs={columnDefs as ColDef<File>[]}defaultColDef={defaultColDef}- loading={isSearchLoading || deleteDocumentMutation.isPending}+ loading={isLoading || deleteDocumentMutation.isPending}ref={gridRef}theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}rowData={gridRows}@@ -867,7 +903,7 @@ function SearchPage() {className="w-full overflow-auto"columnDefs={columnDefs as ColDef<File>[]}defaultColDef={defaultColDef}- loading={isSearchLoading || deleteDocumentMutation.isPending}+ loading={isLoading || deleteDocumentMutation.isPending}ref={gridRef}theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}rowData={gridRows}diff --git a/frontend/app/settings/_components/connector-card.tsx b/frontend/app/settings/_components/connector-card.tsxindex 820441b93..314284ffd 100644--- a/frontend/app/settings/_components/connector-card.tsx+++ b/frontend/app/settings/_components/connector-card.tsx@@ -189,7 +189,7 @@ export default function ConnectorCard({className={cn("w-full cursor-pointer",isCloudBrand- ? "rounded-none border-layer-contextual-foreground bg-layer-contextual text-layer-contextual-foreground hover:bg-primary hover:text-primary-foreground hover:border-primary"+ ? "rounded-none border-primary bg-layer-contextual text-layer-contextual-foreground hover:bg-primary hover:text-primary-foreground hover:border-primary": "group-hover:bg-background group-hover:border-zinc-700 group-hover:text-primary",)}size="sm"diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex 4235c34ab..6d315d85e 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -1,7 +1,13 @@"use client";import { useQueryClient } from "@tanstack/react-query";-import { AlertCircle, ArrowLeft, FolderOpen, RefreshCw } from "lucide-react";+import {+ AlertCircle,+ ArrowLeft,+ FileSearch,+ FolderOpen,+ RefreshCw,+} from "lucide-react";import { useParams, useRouter } from "next/navigation";import { useState } from "react";import { toast } from "sonner";@@ -13,6 +19,7 @@ import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQueryimport { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";import { IngestSettings } from "@/components/cloud-picker/ingest-settings";import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";+import { FileBrowserDialog } from "@/components/file-browser-dialog";import { Button } from "@/components/ui/button";import {Tooltip,@@ -58,6 +65,9 @@ function BucketView({);const [ingestSettings, setIngestSettings] = useSessionIngestSettings();const [isSettingsOpen, setIsSettingsOpen] = useState(false);+ const [browseDialogBucket, setBrowseDialogBucket] = useState<string | null>(+ null,+ );const invalidate = () => {queryClient.invalidateQueries({ queryKey: invalidateQueryKey });@@ -218,6 +228,18 @@ function BucketView({</p>)}</div>+ <Button+ variant="outline"+ size="sm"+ className="shrink-0"+ onClick={(e) => {+ e.stopPropagation();+ setBrowseDialogBucket(bucket.name);+ }}+ >+ <FileSearch size={14} className="mr-1.5" />+ Browse Files+ </Button></div></div>);@@ -256,6 +278,18 @@ function BucketView({</Button></div></div>++ {connector.connectionId && browseDialogBucket && (+ <FileBrowserDialog+ open+ onOpenChange={(open) => {+ if (!open) setBrowseDialogBucket(null);+ }}+ connectorType={connector.type}+ connectionId={connector.connectionId}+ buckets={[browseDialogBucket]}+ />+ )}</>);}diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsxnew file mode 100644index 000000000..0681d339c--- /dev/null+++ b/frontend/components/file-browser-dialog.tsx@@ -0,0 +1,289 @@+"use client";++import { Check, Download, Loader2, Search } from "lucide-react";+import { useCallback, useMemo, useState } from "react";+import { toast } from "sonner";+import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";+import {+ type RemoteFile,+ useBrowseConnectionFiles,+} from "@/app/api/queries/useBrowseConnectionFiles";+import { Badge } from "./ui/badge";+import { Button } from "./ui/button";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "./ui/dialog";+import { Input } from "./ui/input";+import { ScrollArea } from "./ui/scroll-area";++interface FileBrowserDialogProps {+ open: boolean;+ onOpenChange: (open: boolean) => void;+ connectorType: string;+ connectionId: string;+ buckets?: string[];+}++function formatFileSize(bytes: number): string {+ if (bytes === 0) return "0 B";+ const units = ["B", "KB", "MB", "GB"];+ const i = Math.floor(Math.log(bytes) / Math.log(1024));+ return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;+}++export function FileBrowserDialog({+ open,+ onOpenChange,+ connectorType,+ connectionId,+ buckets,+}: FileBrowserDialogProps) {+ const [search, setSearch] = useState("");+ const [selectedBucket, setSelectedBucket] = useState<string | undefined>(+ buckets?.[0],+ );+ const [selectedFileIds, setSelectedFileIds] = useState<Set<string>>(+ new Set(),+ );++ const syncMutation = useSyncConnector();++ const { data, isLoading, error } = useBrowseConnectionFiles(+ {+ connectorType,+ connectionId,+ bucket: selectedBucket,+ search: search || undefined,+ maxFiles: 500,+ },+ { enabled: open },+ );++ const files = data?.files ?? [];++ const toggleFile = useCallback((fileId: string) => {+ setSelectedFileIds((prev) => {+ const next = new Set(prev);+ if (next.has(fileId)) {+ next.delete(fileId);+ } else {+ next.add(fileId);+ }+ return next;+ });+ }, []);++ const toggleAll = useCallback(() => {+ const unIngestedFiles = files.filter((f) => !f.is_ingested);+ if (selectedFileIds.size === unIngestedFiles.length) {+ setSelectedFileIds(new Set());+ } else {+ setSelectedFileIds(new Set(unIngestedFiles.map((f) => f.id)));+ }+ }, [files, selectedFileIds.size]);++ const selectedFiles = useMemo(+ () => files.filter((f) => selectedFileIds.has(f.id)),+ [files, selectedFileIds],+ );++ const handleIngest = useCallback(async () => {+ if (selectedFiles.length === 0) return;++ try {+ await syncMutation.mutateAsync({+ connectorType,+ body: {+ selected_files: selectedFiles.map((f) => ({+ id: f.id,+ name: f.name,+ mimeType: "",+ size: f.size,+ })),+ },+ });++ toast.success("Ingestion started", {+ description: `${selectedFiles.length} file(s) queued for ingestion.`,+ });++ setSelectedFileIds(new Set());+ onOpenChange(false);+ } catch (err) {+ toast.error("Ingestion failed", {+ description: err instanceof Error ? err.message : "Unknown error",+ });+ }+ }, [selectedFiles, connectorType, syncMutation, onOpenChange]);++ const unIngestedCount = files.filter((f) => !f.is_ingested).length;++ return (+ <Dialog open={open} onOpenChange={onOpenChange}>+ <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">+ <DialogHeader>+ <DialogTitle>Browse Files</DialogTitle>+ <DialogDescription>+ Select files to ingest from your {connectorType.replace("_", " ")}{" "}+ connection.+ {data && (+ <span className="ml-1">+ {data.total_ingested} of {data.total_remote} file(s) already+ ingested.+ </span>+ )}+ </DialogDescription>+ </DialogHeader>++ <div className="flex gap-2 items-center">+ {buckets && buckets.length > 1 && (+ <select+ className="border rounded px-2 py-1.5 text-sm bg-background"+ value={selectedBucket || ""}+ onChange={(e) => {+ setSelectedBucket(e.target.value || undefined);+ setSelectedFileIds(new Set());+ }}+ >+ {buckets.map((b) => (+ <option key={b} value={b}>+ {b}+ </option>+ ))}+ </select>+ )}+ <div className="relative flex-1">+ <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />+ <Input+ placeholder="Search files..."+ value={search}+ onChange={(e) => setSearch(e.target.value)}+ className="pl-9"+ />+ </div>+ </div>++ <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">+ {isLoading ? (+ <div className="flex items-center justify-center p-8">+ <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />+ <span className="ml-2 text-muted-foreground">+ Loading files...+ </span>+ </div>+ ) : error ? (+ <div className="p-4 text-destructive text-sm">+ Failed to load files:{" "}+ {error instanceof Error ? error.message : "Unknown error"}+ </div>+ ) : files.length === 0 ? (+ <div className="p-8 text-center text-muted-foreground text-sm">+ No files found.+ </div>+ ) : (+ <div className="divide-y">+ {unIngestedCount > 0 && (+ <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">+ <input+ type="checkbox"+ checked={+ selectedFileIds.size === unIngestedCount &&+ unIngestedCount > 0+ }+ onChange={toggleAll}+ className="h-4 w-4 rounded border-border"+ />+ <span className="text-xs text-muted-foreground">+ {selectedFileIds.size > 0+ ? `${selectedFileIds.size} selected`+ : `Select all (${unIngestedCount})`}+ </span>+ </div>+ )}+ {files.map((file) => (+ <FileRow+ key={file.id}+ file={file}+ selected={selectedFileIds.has(file.id)}+ onToggle={() => toggleFile(file.id)}+ />+ ))}+ </div>+ )}+ </ScrollArea>++ <DialogFooter>+ <Button variant="outline" onClick={() => onOpenChange(false)}>+ Cancel+ </Button>+ <Button+ onClick={handleIngest}+ disabled={selectedFiles.length === 0 || syncMutation.isPending}+ >+ {syncMutation.isPending ? (+ <>+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />+ Ingesting...+ </>+ ) : (+ <>+ <Download className="h-4 w-4 mr-2" />+ Ingest{" "}+ {selectedFiles.length > 0+ ? `${selectedFiles.length} file(s)`+ : "selected"}+ </>+ )}+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ );+}++function FileRow({+ file,+ selected,+ onToggle,+}: {+ file: RemoteFile;+ selected: boolean;+ onToggle: () => void;+}) {+ return (+ <label+ className={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${+ file.is_ingested ? "opacity-60" : ""+ }`}+ >+ <input+ type="checkbox"+ checked={selected || file.is_ingested}+ disabled={file.is_ingested}+ onChange={onToggle}+ className="h-4 w-4 rounded border-border"+ />+ <div className="flex-1 min-w-0">+ <div className="text-sm truncate font-medium">{file.name}</div>+ <div className="text-xs text-muted-foreground flex gap-2">+ {file.bucket && <span>{file.bucket}</span>}+ <span>{formatFileSize(file.size)}</span>+ {file.modified_time && (+ <span>{new Date(file.modified_time).toLocaleDateString()}</span>+ )}+ </div>+ </div>+ {file.is_ingested && (+ <Badge variant="secondary" className="text-xs flex-shrink-0">+ <Check className="h-3 w-3 mr-1" />+ Ingested+ </Badge>+ )}+ </label>+ );+}diff --git a/frontend/components/knowledge-batch-actions-bar.tsx b/frontend/components/knowledge-batch-actions-bar.tsxindex 71e998902..3d730d20d 100644--- a/frontend/components/knowledge-batch-actions-bar.tsx+++ b/frontend/components/knowledge-batch-actions-bar.tsx@@ -17,6 +17,14 @@ export const KnowledgeBatchActionsBar = ({const canDelete = canAny(["knowledge:delete:own", "knowledge:delete:any"]);return (<div className="flex h-12 w-full items-stretch bg-primary text-primary-foreground">+ <button+ type="button"+ aria-label="Cancel selection"+ onClick={onCancel}+ className="flex h-full w-12 flex-shrink-0 items-center justify-center border-r border-primary-foreground/20 transition-colors hover:bg-primary-foreground/10"+ >+ <X className="h-4 w-4" />+ </button><span className="flex items-center px-4 text-sm font-medium">{selectedCount} item{selectedCount !== 1 ? "s" : ""} selected</span>@@ -32,17 +40,9 @@ export const KnowledgeBatchActionsBar = ({}className="flex h-full items-center px-4 text-sm font-medium transition-colors hover:bg-primary-foreground/10 disabled:opacity-50 disabled:cursor-not-allowed">+ <Trash2 className="h-4 w-4" />Delete</button>- <span className="self-center h-4 w-px bg-primary-foreground" />- <button- type="button"- aria-label="Cancel selection"- onClick={onCancel}- className="flex h-full w-12 flex-shrink-0 items-center justify-center transition-colors hover:bg-primary-foreground/10"- >- <X className="h-4 w-4" />- </button></div></div>);diff --git a/src/api/acl.py b/src/api/acl.pynew file mode 100644index 000000000..2b9180f7c--- /dev/null+++ b/src/api/acl.py@@ -0,0 +1,38 @@+from fastapi import Depends+from fastapi.responses import JSONResponse++from config.settings import get_index_name+from dependencies import get_current_user, get_session_manager+from session_manager import User+++async def get_document_acl(+ filename: str,+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+):+ """GET /documents/acl?filename=... — return ACL for a document."""+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)+ response = await opensearch_client.search(+ index=get_index_name(),+ body={+ "query": {"term": {"filename": filename}},+ "size": 1,+ "_source": ["owner", "allowed_users", "allowed_groups"],+ },+ )+ hits = response.get("hits", {}).get("hits", [])+ if not hits:+ return JSONResponse(+ {"error": f"Document '{filename}' not found"},+ status_code=404,+ )++ source = hits[0]["_source"]+ return JSONResponse(+ {+ "owner": source.get("owner"),+ "allowed_users": source.get("allowed_users", []),+ "allowed_groups": source.get("allowed_groups", []),+ }+ )diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 2fceae4d5..e4213d828 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -28,59 +28,46 @@ async def get_synced_file_ids_for_connector("""Query OpenSearch for unique document_id values where connector_type matches.Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.-+Note: Langflow-ingested files may not have document_id stored. In that case,filenames are returned for filename-based filtering during sync."""try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)-+# Query for both document_id and filename aggregationsquery_body = {"size": 0,- "query": {- "term": {- "connector_type": connector_type- }- },+ "query": {"term": {"connector_type": connector_type}},"aggs": {- "unique_document_ids": {- "terms": {- "field": "document_id",- "size": 10000- }- },- "unique_filenames": {- "terms": {- "field": "filename",- "size": 10000- }- }- }+ "unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},+ "unique_filenames": {"terms": {"field": "filename", "size": 10000}},+ },}-- result = await opensearch_client.search(- index=get_index_name(),- body=query_body- )-++ result = await opensearch_client.search(index=get_index_name(), body=query_body)+# Get document_ids (preferred - these are the actual connector file IDs)- doc_id_buckets = result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])+ doc_id_buckets = (+ result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])+ )file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]-+# Get filenames as fallback- filename_buckets = result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])+ filename_buckets = (+ result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])+ )filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]-+logger.debug("Found synced files for connector",connector_type=connector_type,file_ids_count=len(file_ids),filenames_count=len(filenames),)-+return file_ids, filenames-+except Exception as e:logger.error("Failed to get synced file IDs",@@ -90,7 +77,6 @@ async def get_synced_file_ids_for_connector(return [], []-class ConnectorSyncBody(BaseModel):max_files: Optional[int] = Noneselected_files: Optional[List[Any]] = None@@ -140,7 +126,9 @@ async def connector_sync(file_infos = selected_files_rawtry:- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START+ )logger.debug("Starting connector sync",connector_type=connector_type,@@ -201,10 +189,11 @@ async def connector_sync("Starting sync with working connection",connection_id=working_connection.connection_id,)-+if selected_files:# Explicit files selected (e.g., from file picker) - sync those specific filesfrom .documents import _ensure_index_exists+await _ensure_index_exists(jwt_token)task_id = await connector_service.sync_specific_files(working_connection.connection_id,@@ -242,7 +231,10 @@ async def connector_sync(if not all_file_ids:return JSONResponse(- {"status": "no_files", "message": "No files found in the selected buckets."},+ {+ "status": "no_files",+ "message": "No files found in the selected buckets.",+ },status_code=200,)task_id = await connector_service.sync_specific_files(@@ -308,7 +300,9 @@ async def connector_sync(filename_filter=set(existing_filenames),)task_ids = [task_id]- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE+ )return JSONResponse({"task_ids": task_ids,@@ -321,7 +315,9 @@ async def connector_sync(except Exception as e:logger.error("Connector sync failed", error=str(e))- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED+ )return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)@@ -340,25 +336,31 @@ async def connector_status(# Get the connector for each connection and verify authenticationconnection_details = {}verified_active_connections = []-+for connection in connections:try:connector = await connector_service._get_connector(connection.connection_id)if connector is not None:# Actually verify the connection by trying to authenticateis_authenticated = await connector.authenticate()-+# Get base URL if available (for SharePoint/OneDrive connectors)base_url = None- if hasattr(connector, 'base_url'):+ if hasattr(connector, "base_url"):base_url = connector.base_url- logger.debug(f"connector_status: Got base_url from connector.base_url: {base_url}")- elif hasattr(connector, 'sharepoint_url'):+ logger.debug(+ f"connector_status: Got base_url from connector.base_url: {base_url}"+ )+ elif hasattr(connector, "sharepoint_url"):base_url = connector.sharepoint_url # Backward compatibility- logger.debug(f"connector_status: Got base_url from connector.sharepoint_url: {base_url}")+ logger.debug(+ f"connector_status: Got base_url from connector.sharepoint_url: {base_url}"+ )else:- logger.debug(f"connector_status: Connector has no base_url or sharepoint_url attribute")-+ logger.debug(+ f"connector_status: Connector has no base_url or sharepoint_url attribute"+ )+connection_details[connection.connection_id] = {"client_id": connector.get_client_id(),"is_authenticated": is_authenticated,@@ -397,8 +399,13 @@ async def connector_status("connection_id": conn.connection_id,"name": conn.name,"client_id": connection_details.get(conn.connection_id, {}).get("client_id"),- "is_active": conn.is_active and connection_details.get(conn.connection_id, {}).get("is_authenticated", False),- "is_authenticated": connection_details.get(conn.connection_id, {}).get("is_authenticated", False),+ "is_active": conn.is_active+ and connection_details.get(conn.connection_id, {}).get(+ "is_authenticated", False+ ),+ "is_authenticated": connection_details.get(conn.connection_id, {}).get(+ "is_authenticated", False+ ),"base_url": connection_details.get(conn.connection_id, {}).get("base_url"),"created_at": conn.created_at.isoformat(),"last_sync": conn.last_sync.isoformat() if conn.last_sync else None,@@ -428,10 +435,10 @@ async def connector_webhook(config=temp_config,)try:- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_RECV)- temp_connector = connector_service.connection_manager._create_connector(- temp_connection+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_RECV)+ temp_connector = connector_service.connection_manager._create_connector(temp_connection)validation_response = temp_connector.handle_webhook_validation(request.method, dict(request.headers), dict(request.query_params))@@ -466,32 +473,22 @@ async def connector_webhook(# Extract channel/subscription ID using connector-specific methodtry:- temp_connector = connector_service.connection_manager._create_connector(- temp_connection- )+ temp_connector = connector_service.connection_manager._create_connector(temp_connection)channel_id = temp_connector.extract_webhook_channel_id(payload, headers)except (NotImplementedError, ValueError):channel_id = Noneif not channel_id:- logger.warning(- "No channel ID found in webhook", connector_type=connector_type- )+ logger.warning("No channel ID found in webhook", connector_type=connector_type)return JSONResponse({"status": "ignored", "reason": "no_channel_id"})# Find the specific connection for this webhook- connection = (- await connector_service.connection_manager.get_connection_by_webhook_id(- channel_id- )+ connection = await connector_service.connection_manager.get_connection_by_webhook_id(+ channel_id)if not connection or not connection.is_active:- logger.info(- "Unknown webhook channel, will auto-expire", channel_id=channel_id- )- return JSONResponse(- {"status": "ignored_unknown_channel", "channel_id": channel_id}- )+ logger.info("Unknown webhook channel, will auto-expire", channel_id=channel_id)+ return JSONResponse({"status": "ignored_unknown_channel", "channel_id": channel_id})# Process webhook for the specific connectiontry:@@ -502,9 +499,7 @@ async def connector_webhook("Could not get connector for connection",connection_id=connection.connection_id,)- return JSONResponse(- {"status": "error", "reason": "connector_not_found"}- )+ return JSONResponse({"status": "error", "reason": "connector_not_found"})# Let the connector handle the webhook and return affected file IDsaffected_files = await connector.handle_webhook(payload)@@ -575,10 +570,11 @@ async def connector_webhook(except Exception as e:logger.error("Webhook processing failed", error=str(e))- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_FAILED)- return JSONResponse(- {"error": f"Webhook processing failed: {str(e)}"}, status_code=500+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_FAILED)+ return JSONResponse({"error": f"Webhook processing failed: {str(e)}"}, status_code=500)+async def connector_disconnect(connector_type: str,@@ -605,7 +601,7 @@ async def connector_disconnect(try:# Get the connector to cleanup any subscriptionsconnector = await connector_service._get_connector(connection.connection_id)- if connector and hasattr(connector, 'cleanup_subscription'):+ if connector and hasattr(connector, "cleanup_subscription"):subscription_id = connection.config.get("webhook_channel_id")if subscription_id:try:@@ -659,6 +655,7 @@ async def connector_disconnect(# ---------------------------------------------------------------------------+async def sync_all_connectors(connector_service=Depends(get_connector_service),session_manager=Depends(get_session_manager),@@ -668,12 +665,14 @@ async def sync_all_connectors(Sync files from all active cloud connector connections."""try:- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START+ )jwt_token = user.jwt_token# Cloud connector types to synccloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]-+all_task_ids = []synced_connectors = []skipped_connectors = []@@ -688,7 +687,7 @@ async def sync_all_connectors(session_manager=session_manager,jwt_token=jwt_token,)-+if not existing_file_ids and not existing_filenames:logger.debug("No existing files in OpenSearch for connector type, skipping",@@ -760,14 +759,16 @@ async def sync_all_connectors(jwt_token=jwt_token,filename_filter=set(existing_filenames),)-+all_task_ids.append(task_id)synced_connectors.append(connector_type)logger.info("Started sync for connector type",connector_type=connector_type,task_id=task_id,- file_count=len(existing_file_ids) if existing_file_ids else len(existing_filenames),+ file_count=len(existing_file_ids)+ if existing_file_ids+ else len(existing_filenames),)except Exception as e:@@ -793,7 +794,9 @@ async def sync_all_connectors(status_code=404,)- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE+ )return JSONResponse({"task_ids": all_task_ids,@@ -808,7 +811,9 @@ async def sync_all_connectors(except Exception as e:logger.error("Sync all connectors failed", error=str(e))- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)+ await TelemetryClient.send_event(+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED+ )return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)@@ -832,7 +837,9 @@ async def connector_token(connector = await connector_service._get_connector(connection_id)if not connector:return JSONResponse(- {"error": f"Connector not available - authentication may have failed for {url_connector_type}"},+ {+ "error": f"Connector not available - authentication may have failed for {url_connector_type}"+ },status_code=404,)@@ -864,7 +871,10 @@ async def connector_token(try:if connector.oauth.creds.expiry:import time- expires_in = max(0, int(connector.oauth.creds.expiry.timestamp() - time.time()))++ expires_in = max(+ 0, int(connector.oauth.creds.expiry.timestamp() - time.time())+ )except Exception:expires_in = None@@ -911,12 +921,114 @@ async def connector_token(return JSONResponse({"access_token": access_token, "expires_in": None})except ValueError as e:# Typical when acquire_token_silent fails (e.g., needs re-auth)- return JSONResponse({"error": f"Failed to get access token: {str(e)}"}, status_code=401)+ return JSONResponse(+ {"error": f"Failed to get access token: {str(e)}"}, status_code=401+ )except Exception as e:return JSONResponse({"error": f"Authentication error: {str(e)}"}, status_code=500)- return JSONResponse({"error": "Token not available for this connector type"}, status_code=400)+ return JSONResponse(+ {"error": "Token not available for this connector type"}, status_code=400+ )except Exception as e:logger.error("Error getting connector token", exc_info=True)return JSONResponse({"error": str(e)}, status_code=500)+++async def browse_connection_files(+ connector_type: str,+ connection_id: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+ bucket: Optional[str] = None,+ search: Optional[str] = None,+ page_token: Optional[str] = None,+ max_files: int = 100,+):+ """+ Browse remote files in a connector with ingestion status.++ Lists files from the remote source (e.g., S3 bucket) and marks each+ as ingested or not by cross-referencing with OpenSearch.+ """+ try:+ connector = await connector_service.get_connector(connection_id)+ if not connector:+ return JSONResponse(+ {"error": "Connection not found or connector unavailable"},+ status_code=404,+ )++ if not await connector.authenticate():+ return JSONResponse(+ {"error": "Connector authentication failed"},+ status_code=401,+ )++ # Temporarily override bucket filter if specified+ original_buckets = None+ if bucket and hasattr(connector, "bucket_names"):+ original_buckets = connector.bucket_names+ connector.bucket_names = [bucket]++ try:+ files_result = await connector.list_files(page_token=page_token, max_files=max_files)+ finally:+ if original_buckets is not None:+ connector.bucket_names = original_buckets++ remote_files = files_result.get("files", [])+ next_page_token = files_result.get("next_page_token")++ # Filter by filename search if provided+ if search:+ search_lower = search.lower()+ remote_files = [f for f in remote_files if search_lower in f.get("name", "").lower()]++ # Get already-ingested file IDs from OpenSearch+ ingested_ids, ingested_filenames = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,+ jwt_token=user.jwt_token,+ )+ ingested_set = set(ingested_ids) | set(ingested_filenames)++ # Merge ingestion status into remote file list+ enriched_files = []+ for f in remote_files:+ is_ingested = f.get("id", "") in ingested_set or f.get("name", "") in ingested_set+ enriched_files.append(+ {+ "id": f.get("id", ""),+ "name": f.get("name", ""),+ "bucket": f.get("bucket", ""),+ "key": f.get("key", ""),+ "size": f.get("size", 0),+ "modified_time": f.get("modified_time", ""),+ "is_ingested": is_ingested,+ }+ )++ return JSONResponse(+ {+ "files": enriched_files,+ "next_page_token": next_page_token,+ "total_remote": len(enriched_files),+ "total_ingested": sum(1 for f in enriched_files if f["is_ingested"]),+ }+ )++ except Exception as e:+ logger.error(+ "Failed to browse connection files",+ connector_type=connector_type,+ connection_id=connection_id,+ error=str(e),+ )+ return JSONResponse(+ {"error": f"Failed to browse files: {str(e)}"},+ status_code=500,+ )diff --git a/src/api/documents.py b/src/api/documents.pyindex 416e146bd..ce9b6275b 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -37,9 +37,32 @@ async def delete_documents_by_filename_core()try:- opensearch_client = session_manager.get_user_opensearch_client(- user_id, jwt_token+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)++ # Owner check: only the document owner may delete+ check = await opensearch_client.search(+ index=get_index_name(),+ body={+ "query": {"term": {"filename": normalized_filename}},+ "size": 1,+ "_source": ["owner"],+ },)+ hits = check.get("hits", {}).get("hits", [])+ if hits:+ doc_owner = hits[0]["_source"].get("owner")+ if doc_owner and doc_owner != user_id:+ return (+ {+ "success": False,+ "deleted_chunks": 0,+ "filename": normalized_filename,+ "message": None,+ "error": "Access denied: only the document owner can delete this file",+ },+ 403,+ )+delete_query = build_filename_delete_body(normalized_filename)result = await opensearch_client.delete_by_query(index=get_index_name(),@@ -122,9 +145,7 @@ async def check_filename_exists(jwt_token = user.jwt_tokentry:- opensearch_client = session_manager.get_user_opensearch_client(- user.user_id, jwt_token- )+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)from utils.opensearch_queries import build_filename_search_bodyfrom utils.file_utils import get_filename_aliases@@ -139,10 +160,7 @@ async def check_filename_exists(try:for candidate in candidate_filenames:search_body = build_filename_search_body(candidate, size=1, source=["filename"])- response = await opensearch_client.search(- index=get_index_name(),- body=search_body- )+ response = await opensearch_client.search(index=get_index_name(), body=search_body)hits = response.get("hits", {}).get("hits", [])if hits:exists = True@@ -160,7 +178,9 @@ async def check_filename_exists(logger.error("Error checking filename existence", filename=filename, error=str(e))error_str = str(e)if "AuthenticationException" in error_str:- return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)+ return JSONResponse(+ {"error": "Access denied: insufficient permissions"}, status_code=403+ )else:return JSONResponse({"error": str(e)}, status_code=500)@@ -169,9 +189,9 @@ async def delete_documents_by_filename(body: DeleteDocumentBody,session_manager=Depends(get_session_manager),user: User = Depends(require_permission("knowledge:delete:own")),- ):+):"""Delete all documents with a specific filename"""- payload, status_code =await delete_documents_by_filename_core(+ payload, status_code = await delete_documents_by_filename_core(filename=body.filename,session_manager=session_manager,user_id=user.user_id,diff --git a/src/api/files.py b/src/api/files.pynew file mode 100644index 000000000..b88fcf636--- /dev/null+++ b/src/api/files.py@@ -0,0 +1,89 @@+"""+File listing and search API endpoints.++Provides server-side file-level views over the chunk-based OpenSearch index,+replacing the client-side chunk-to-file aggregation pattern.+"""++from typing import Optional++from fastapi import Depends, Query+from fastapi.responses import JSONResponse++from dependencies import get_session_manager, get_current_user+from session_manager import User+from utils.logging_config import get_logger++logger = get_logger(__name__)+++def _get_file_service(session_manager=Depends(get_session_manager)):+ from services.file_service import FileService++ return FileService(session_manager=session_manager)+++async def list_files(+ page: int = Query(1, ge=1, description="Page number"),+ page_size: int = Query(25, ge=1, le=100, description="Items per page"),+ sort_by: str = Query("filename", description="Sort field"),+ sort_order: str = Query("asc", regex="^(asc|desc)$", description="Sort order"),+ connector_type: Optional[str] = Query(None, description="Filter by connector type"),+ mimetype: Optional[str] = Query(None, description="Filter by MIME type"),+ owner: Optional[str] = Query(None, description="Filter by owner"),+ search: Optional[str] = Query(None, description="Search filename"),+ file_service=Depends(_get_file_service),+ user: User = Depends(get_current_user),+):+ """List ingested files with pagination, filtering, and sorting."""+ try:+ result = await file_service.list_files(+ user_id=user.user_id,+ jwt_token=user.jwt_token,+ page=page,+ page_size=page_size,+ sort_by=sort_by,+ sort_order=sort_order,+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ search=search,+ )+ return JSONResponse(result)+ except Exception as e:+ logger.error("Failed to list files", error=str(e))+ return JSONResponse(+ {"error": "Failed to list files", "detail": str(e)},+ status_code=500,+ )+++async def search_files(+ q: str = Query(..., min_length=1, description="Search query"),+ page: int = Query(1, ge=1, description="Page number"),+ page_size: int = Query(25, ge=1, le=100, description="Items per page"),+ connector_type: Optional[str] = Query(None, description="Filter by connector type"),+ mimetype: Optional[str] = Query(None, description="Filter by MIME type"),+ owner: Optional[str] = Query(None, description="Filter by owner"),+ file_service=Depends(_get_file_service),+ user: User = Depends(get_current_user),+):+ """Search files by name with fuzzy/partial matching."""+ try:+ result = await file_service.search_files(+ user_id=user.user_id,+ jwt_token=user.jwt_token,+ query=q,+ page=page,+ page_size=page_size,+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ )+ return JSONResponse(result)+ except Exception as e:+ logger.error("Failed to search files", error=str(e))+ return JSONResponse(+ {"error": "Failed to search files", "detail": str(e)},+ status_code=500,+ )diff --git a/src/app/routes/internal.py b/src/app/routes/internal.pyindex 78bc8ec63..c1de1914f 100644--- a/src/app/routes/internal.py+++ b/src/app/routes/internal.py@@ -14,6 +14,7 @@ from api import (connectors,docling,documents,+ files,flows,knowledge_filter,langflow_files,@@ -89,6 +90,10 @@ def register_internal_routes(app: FastAPI):# Search endpointapp.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])+ # File listing/search endpoints+ app.add_api_route("/files", files.list_files, methods=["GET"], tags=["internal"])+ app.add_api_route("/files/search", files.search_files, methods=["GET"], tags=["internal"])+# Knowledge Filter endpointsapp.add_api_route("/knowledge-filter",diff --git a/src/connectors/aws_s3/support.py b/src/connectors/aws_s3/support.pyindex ccaa069b7..1dbbef99e 100644--- a/src/connectors/aws_s3/support.py+++ b/src/connectors/aws_s3/support.py@@ -24,14 +24,13 @@ def build_s3_config(({}, error_message) on validation failure"""access_key = (- body.access_key- or os.getenv("AWS_ACCESS_KEY_ID")- or existing_config.get("access_key")+ body.access_key or os.getenv("AWS_ACCESS_KEY_ID") or existing_config.get("access_key"))secret_key = (- body.secret_key- or os.getenv("AWS_SECRET_ACCESS_KEY")- or existing_config.get("secret_key")+ body.secret_key or os.getenv("AWS_SECRET_ACCESS_KEY") or existing_config.get("secret_key")+ )+ session_token = (+ body.session_token or os.getenv("AWS_SESSION_TOKEN") or existing_config.get("session_token"))if not access_key or not secret_key:@@ -41,11 +40,19 @@ def build_s3_config("access_key": access_key.strip(),"secret_key": secret_key.strip(),}+ if session_token:+ conn_config["session_token"] = session_token.strip()if body.endpoint_url:conn_config["endpoint_url"] = body.endpoint_url.strip()+ elif existing_config.get("endpoint_url"):+ conn_config["endpoint_url"] = existing_config["endpoint_url"]if body.region:conn_config["region"] = body.region.strip()+ elif existing_config.get("region"):+ conn_config["region"] = existing_config["region"]if body.bucket_names is not None:conn_config["bucket_names"] = body.bucket_names+ elif existing_config.get("bucket_names") is not None:+ conn_config["bucket_names"] = existing_config["bucket_names"]return conn_config, Nonediff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.pyindex 8646d72aa..79695bf45 100644--- a/src/connectors/connection_manager.py+++ b/src/connectors/connection_manager.py@@ -39,9 +39,12 @@ class ConnectionConfig:class ConnectionManager:"""Manages multiple connector connections with persistence"""- def __init__(self, connections_file: str = None):- from config.paths import get_data_file- self.connections_file = Path(connections_file or get_data_file("connections.json"))+ def __init__(self, connections_file: Optional[str] = None):+ if connections_file is None:+ data_dir = Path(os.getenv("OPENRAG_DATA_PATH", "data"))+ self.connections_file = data_dir / "connections.json"+ else:+ self.connections_file = Path(connections_file)# Ensure data directory existsself.connections_file.parent.mkdir(parents=True, exist_ok=True)self.connections: Dict[str, ConnectionConfig] = {}@@ -50,15 +53,24 @@ class ConnectionManager:async def load_connections(self):"""Load connections from persistent storage"""from utils.encryption import decrypt_secret, get_master_secret-+needs_encryption_upgrade = Falsedecryption_failed = Falsesecret_keys = {- "api_key", "hmac_secret_key", "secret_key", "client_secret",- "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",- "access_key", "hmac_access_key", "service_instance_id","basic_credentials"+ "api_key",+ "hmac_secret_key",+ "secret_key",+ "client_secret",+ "aws_secret_access_key",+ "ibm_api_key",+ "access_token",+ "refresh_token",+ "access_key",+ "hmac_access_key",+ "service_instance_id",+ "basic_credentials",}-+if self.connections_file.exists():async with aiofiles.open(self.connections_file, "r") as f:data = json.loads(await f.read())@@ -70,27 +82,25 @@ class ConnectionManager:if isinstance(v, dict) and v.get("algorithm") == "AES-256-GCM":try:tenant = conn_data.get("user_id") or "openrag"- conn_data["config"][k] = decrypt_secret(v, expected_tenant_id=tenant)+ conn_data["config"][k] = decrypt_secret(+ v, expected_tenant_id=tenant+ )except ValueError as e:logger.error(f"Failed to decrypt connection secret {k}: {e}")decryption_failed = Trueelif k in secret_keys and isinstance(v, str) and v:if get_master_secret() is not None:needs_encryption_upgrade = True-+# Convert datetime strings back to datetime objectsif conn_data.get("created_at"):- conn_data["created_at"] = datetime.fromisoformat(- conn_data["created_at"]- )+ conn_data["created_at"] = datetime.fromisoformat(conn_data["created_at"])if conn_data.get("last_sync"):- conn_data["last_sync"] = datetime.fromisoformat(- conn_data["last_sync"]- )+ conn_data["last_sync"] = datetime.fromisoformat(conn_data["last_sync"])config = ConnectionConfig(**conn_data)self.connections[config.connection_id] = config-+if needs_encryption_upgrade:if decryption_failed:logger.warning(@@ -111,24 +121,34 @@ class ConnectionManager:async def save_connections(self):"""Save connections to persistent storage"""from utils.encryption import encrypt_secret+secret_keys = {- "api_key", "hmac_secret_key", "secret_key", "client_secret",- "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",- "access_key", "hmac_access_key", "service_instance_id", "basic_credentials"+ "api_key",+ "hmac_secret_key",+ "secret_key",+ "client_secret",+ "aws_secret_access_key",+ "ibm_api_key",+ "access_token",+ "refresh_token",+ "access_key",+ "hmac_access_key",+ "service_instance_id",+ "basic_credentials",}-+data = {"connections": []}for config in self.connections.values():conn_data = asdict(config)-+# Encrypt sensitive fields in configif "config" in conn_data and isinstance(conn_data["config"], dict):for k, v in conn_data["config"].items():if k in secret_keys and isinstance(v, str):tenant_id = conn_data.get("user_id") or "openrag"conn_data["config"][k] = encrypt_secret(v, tenant_id=tenant_id)-+# Convert datetime objects to stringsif conn_data.get("created_at"):conn_data["created_at"] = conn_data["created_at"].isoformat()@@ -185,57 +205,63 @@ class ConnectionManager:async def cleanup_duplicate_connections(self, remove_duplicates=False):"""- Clean up duplicate connections, keeping only the most recent connection+ Clean up duplicate connections, keeping only the most recent connectionper provider per user-+Args:remove_duplicates: If True, physically removes duplicates from connections.jsonIf False (default), just deactivates them"""logger.info("Starting cleanup of duplicate connections")-+# Group connections by (connector_type, user_id)grouped_connections = {}-+for connection_id, connection in self.connections.items():if not connection.is_active:continue # Skip inactive connections-+key = (connection.connector_type, connection.user_id)-+if key not in grouped_connections:grouped_connections[key] = []-+grouped_connections[key].append((connection_id, connection))-+# For each group, keep only the most recent connectionconnections_to_remove = []-+for (connector_type, user_id), connections in grouped_connections.items():if len(connections) <= 1:continue # No duplicates-- logger.info(f"Found {len(connections)} duplicate connections for {connector_type}, user {user_id}")-++ logger.info(+ f"Found {len(connections)} duplicate connections for {connector_type}, user {user_id}"+ )+# Sort by created_at, keep the most recentconnections.sort(key=lambda x: x[1].created_at, reverse=True)-+# Keep the first (most recent), remove/deactivate the restfor connection_id, connection in connections[1:]:connections_to_remove.append((connection_id, connection))- logger.info(f"Marking connection {connection_id} for {'removal' if remove_duplicates else 'deactivation'}")-+ logger.info(+ f"Marking connection {connection_id} for {'removal' if remove_duplicates else 'deactivation'}"+ )+# Remove or deactivate duplicate connectionsfor connection_id, connection in connections_to_remove:if remove_duplicates:await self.delete_connection(connection_id) # Handles token cleanupelse:await self.deactivate_connection(connection_id)-+action = "Removed" if remove_duplicates else "Deactivated"- logger.info(f"Cleanup complete. {action} {len(connections_to_remove)} duplicate connections")+ logger.info(+ f"Cleanup complete. {action} {len(connections_to_remove)} duplicate connections"+ )return len(connections_to_remove)-+async def update_connection(self,connection_id: str,@@ -285,10 +311,10 @@ class ConnectionManager:user_id: Optional[str] = None,) -> str:"""Create a new connection configuration, ensuring only one per provider per user"""-+# Check if we already have an active connection for this provider and userexisting_connection = await self._get_existing_connection(connector_type, user_id)-+if existing_connection:# Check if the existing connection has a valid tokentry:@@ -296,26 +322,25 @@ class ConnectionManager:if await connector.authenticate():logger.info(f"Using existing valid connection for {connector_type}",- connection_id=existing_connection.connection_id+ connection_id=existing_connection.connection_id,)# Update the existing connection with new config if neededif config != existing_connection.config:logger.info("Updating existing connection config")await self.update_connection(- existing_connection.connection_id,- config=config+ existing_connection.connection_id, config=config)return existing_connection.connection_idexcept Exception as e:logger.warning(f"Existing connection authentication failed: {e}",- connection_id=existing_connection.connection_id+ connection_id=existing_connection.connection_id,)# If authentication fails, we'll create a new connection and clean up the old one-+# Create new connectionconnection_id = str(uuid.uuid4())-+connection_config = ConnectionConfig(connection_id=connection_id,connector_type=connector_type,@@ -325,10 +350,10 @@ class ConnectionManager:)self.connections[connection_id] = connection_config-+# Clean up duplicates (will keep the newest, which is the one we just created)await self.cleanup_duplicate_connections(remove_duplicates=True)-+await self.save_connections()return connection_id@@ -368,10 +393,7 @@ class ConnectionManager:connector = self.active_connectors[connection_id]# Try to cleanup subscriptions if applicabletry:- if (- hasattr(connector, "webhook_channel_id")- and connector.webhook_channel_id- ):+ if hasattr(connector, "webhook_channel_id") and connector.webhook_channel_id:await connector.cleanup_subscription(connector.webhook_channel_id)except:pass # Best effort cleanup@@ -385,7 +407,7 @@ class ConnectionManager:async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:"""Get an active connector instance"""logger.debug(f"Getting connector for connection_id: {connection_id}")-+# Return cached connector if availableif connection_id in self.active_connectors:connector = self.active_connectors[connection_id]@@ -405,11 +427,11 @@ class ConnectionManager:logger.debug(f"Creating connector for {connection_config.connector_type}")connector = self._create_connector(connection_config)-+logger.debug(f"Attempting authentication for {connection_id}")auth_result = await connector.authenticate()logger.debug(f"Authentication result for {connection_id}: {auth_result}")-+if auth_result:self.active_connectors[connection_id] = connector# ... rest of the method@@ -461,9 +483,7 @@ class ConnectionManager:},}- def _has_saved_credentials_for_user(- self, connector_type: str, user_id: Optional[str]- ) -> bool:+ def _has_saved_credentials_for_user(self, connector_type: str, user_id: Optional[str]) -> bool:"""Check if user has an active saved connection with usable credentials."""for connection in self.connections.values():if connection.connector_type != connector_type or not connection.is_active:@@ -479,9 +499,7 @@ class ConnectionManager:continuereturn False- def _is_connector_available(- self, connector_type: str, user_id: Optional[str] = None- ) -> bool:+ def _is_connector_available(self, connector_type: str, user_id: Optional[str] = None) -> bool:"""Check whether connector is available for use by the given user."""try:temp_config = ConnectionConfig(@@ -549,14 +567,12 @@ class ConnectionManager:return Truereturn False-+async def get_connection(self, connection_id: str) -> Optional[ConnectionConfig]:"""Get connection configuration"""return self.connections.get(connection_id)- async def get_connection_by_webhook_id(- self, webhook_id: str- ) -> Optional[ConnectionConfig]:+ async def get_connection_by_webhook_id(self, webhook_id: str) -> Optional[ConnectionConfig]:"""Find a connection by its webhook/subscription ID"""for connection in self.connections.values():# Check if the webhook ID is stored in the connection config@@ -575,12 +591,10 @@ class ConnectionManager:):"""Setup webhook subscription if not already configured"""# Check if webhook is already set up- if connection_config.config.get(- "webhook_channel_id"- ) or connection_config.config.get("subscription_id"):- logger.info(- "Webhook subscription already exists", connection_id=connection_id- )+ if connection_config.config.get("webhook_channel_id") or connection_config.config.get(+ "subscription_id"+ ):+ logger.info("Webhook subscription already exists", connection_id=connection_id)return# Check if webhook URL is configured@@ -598,9 +612,7 @@ class ConnectionManager:# Store the subscription and resource IDs in connection configconnection_config.config["webhook_channel_id"] = subscription_id- connection_config.config["subscription_id"] = (- subscription_id # Alternative field- )+ connection_config.config["subscription_id"] = subscription_id # Alternative fieldif getattr(connector, "webhook_resource_id", None):connection_config.config["resource_id"] = connector.webhook_resource_iddiff --git a/src/services/file_service.py b/src/services/file_service.pynew file mode 100644index 000000000..d94b25160--- /dev/null+++ b/src/services/file_service.py@@ -0,0 +1,237 @@+"""+Service for file-level listing and search via OpenSearch aggregations.++Aggregates document chunks by filename to produce file-level views,+with support for pagination, filtering, sorting, and fuzzy search.+"""++from typing import Any, Dict, List, Optional++from config.settings import get_index_name+from utils.logging_config import get_logger++logger = get_logger(__name__)+++class FileService:+ """Provides file-level views over the chunk-based OpenSearch index."""++ def __init__(self, session_manager=None):+ self.session_manager = session_manager++ async def list_files(+ self,+ user_id: str,+ jwt_token: str = None,+ page: int = 1,+ page_size: int = 25,+ sort_by: str = "filename",+ sort_order: str = "asc",+ connector_type: Optional[str] = None,+ mimetype: Optional[str] = None,+ owner: Optional[str] = None,+ search: Optional[str] = None,+ ) -> Dict[str, Any]:+ """+ List ingested files with server-side pagination, filtering, and sorting.++ Aggregates chunks by filename using OpenSearch terms aggregation,+ then paginates and sorts the resulting file list in-memory.+ """+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)++ query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)+ agg_body = self._build_file_aggregation(query)++ try:+ result = await opensearch_client.search(+ index=get_index_name(),+ body=agg_body,+ )+ except Exception as e:+ logger.error("Failed to list files", error=str(e))+ return {"files": [], "total": 0, "page": page, "page_size": page_size}++ files = self._parse_aggregation_buckets(result)+ files = self._sort_files(files, sort_by, sort_order)++ total = len(files)+ start = (page - 1) * page_size+ end = start + page_size+ paginated = files[start:end]++ return {+ "files": paginated,+ "total": total,+ "page": page,+ "page_size": page_size,+ }++ async def search_files(+ self,+ user_id: str,+ jwt_token: str = None,+ query: str = "",+ page: int = 1,+ page_size: int = 25,+ connector_type: Optional[str] = None,+ mimetype: Optional[str] = None,+ owner: Optional[str] = None,+ ) -> Dict[str, Any]:+ """+ Search files by name with fuzzy/prefix matching.++ Uses wildcard, match_phrase_prefix, and fuzzy queries on the+ filename field, then aggregates matching chunks into file-level results.+ """+ return await self.list_files(+ user_id=user_id,+ jwt_token=jwt_token,+ page=page,+ page_size=page_size,+ sort_by="filename",+ sort_order="asc",+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ search=query,+ )++ def _build_filter_query(+ self,+ user_id: str,+ connector_type: Optional[str] = None,+ mimetype: Optional[str] = None,+ owner: Optional[str] = None,+ search: Optional[str] = None,+ ) -> Dict[str, Any]:+ """Build the bool query with optional filters + filename search."""+ must = []+ filter_clauses = []++ if connector_type:+ filter_clauses.append({"term": {"connector_type": connector_type}})+ if mimetype:+ filter_clauses.append({"term": {"mimetype": mimetype}})+ if owner:+ filter_clauses.append({"term": {"owner": owner}})++ if search:+ # Combine wildcard (partial), prefix, and fuzzy for flexible matching+ must.append(+ {+ "bool": {+ "should": [+ {"wildcard": {"filename": {"value": f"*{search.lower()}*"}}},+ {"prefix": {"filename": search.lower()}},+ ],+ "minimum_should_match": 1,+ }+ }+ )++ query = {"bool": {"filter": filter_clauses}}+ if must:+ query["bool"]["must"] = must++ return query++ def _build_file_aggregation(self, query: Dict[str, Any]) -> Dict[str, Any]:+ """Build the OpenSearch aggregation body for file-level grouping."""+ return {+ "size": 0,+ "query": query,+ "aggs": {+ "files": {+ "terms": {+ "field": "filename",+ "size": 10000,+ },+ "aggs": {+ "file_metadata": {+ "top_hits": {+ "size": 1,+ "_source": [+ "document_id",+ "filename",+ "mimetype",+ "file_size",+ "source_url",+ "owner",+ "owner_name",+ "owner_email",+ "connector_type",+ "embedding_model",+ "embedding_dimensions",+ "indexed_time",+ "allowed_users",+ "allowed_groups",+ ],+ "sort": [{"indexed_time": {"order": "desc"}}],+ }+ },+ "chunk_count": {"value_count": {"field": "_id"}},+ },+ }+ },+ }++ def _parse_aggregation_buckets(self, result: Dict[str, Any]) -> List[Dict[str, Any]]:+ """Parse OpenSearch aggregation buckets into file dicts."""+ buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])++ files = []+ for bucket in buckets:+ hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])+ if not hits:+ continue++ source = hits[0].get("_source", {})+ files.append(+ {+ "filename": bucket["key"],+ "document_id": source.get("document_id", ""),+ "mimetype": source.get("mimetype", ""),+ "file_size": source.get("file_size", 0),+ "source_url": source.get("source_url", ""),+ "owner": source.get("owner", ""),+ "owner_name": source.get("owner_name", ""),+ "owner_email": source.get("owner_email", ""),+ "connector_type": source.get("connector_type", ""),+ "embedding_model": source.get("embedding_model", ""),+ "embedding_dimensions": source.get("embedding_dimensions"),+ "indexed_time": source.get("indexed_time", ""),+ "chunk_count": bucket.get("chunk_count", {}).get("value", 0),+ "allowed_users": source.get("allowed_users", []),+ "allowed_groups": source.get("allowed_groups", []),+ }+ )++ return files++ def _sort_files(+ self,+ files: List[Dict[str, Any]],+ sort_by: str,+ sort_order: str,+ ) -> List[Dict[str, Any]]:+ """Sort file list by the given field."""+ valid_sort_fields = {+ "filename",+ "file_size",+ "mimetype",+ "indexed_time",+ "connector_type",+ "chunk_count",+ "owner",+ }+ if sort_by not in valid_sort_fields:+ sort_by = "filename"++ reverse = sort_order.lower() == "desc"++ return sorted(+ files,+ key=lambda f: f.get(sort_by) or "",+ reverse=reverse,+ )diff --git a/src/services/search_service.py b/src/services/search_service.pyindex 0a55eb01f..b4a0f8935 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -118,9 +118,7 @@ class SearchService:available_models = []failed_models: list = []- opensearch_client = self.session_manager.get_user_opensearch_client(- user_id, jwt_token- )+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)if not is_wildcard_match_all:# Build filter clauses first so we can use them in model detection@@ -141,9 +139,7 @@ class SearchService:if len(values) == 0:# Empty array means "match nothing" - use impossible filter- filter_clauses.append(- {"term": {field_name: "__IMPOSSIBLE_VALUE__"}}- )+ filter_clauses.append({"term": {field_name: "__IMPOSSIBLE_VALUE__"}})elif len(values) == 1:# Single value filterfilter_clauses.append({"term": {field_name: values[0]}})@@ -156,27 +152,22 @@ class SearchService:agg_query = {"size": 0,"aggs": {- "embedding_models": {- "terms": {- "field": "embedding_model",- "size": 10- }- }- }+ "embedding_models": {"terms": {"field": "embedding_model", "size": 10}}+ },}# Apply filters to model detection if any existif filter_clauses:- agg_query["query"] = {- "bool": {- "filter": filter_clauses- }- }+ agg_query["query"] = {"bool": {"filter": filter_clauses}}agg_result = await opensearch_client.search(index=get_index_name(), body=agg_query, params={"terminate_after": 0})- buckets = agg_result.get("aggregations", {}).get("embedding_models", {}).get("buckets", [])+ buckets = (+ agg_result.get("aggregations", {})+ .get("embedding_models", {})+ .get("buckets", [])+ )available_models = [b["key"] for b in buckets if b["key"]]if not available_models:@@ -187,10 +178,12 @@ class SearchService:"Detected embedding models in corpus",available_models=available_models,model_counts={b["key"]: b["doc_count"] for b in buckets},- with_filters=len(filter_clauses) > 0+ with_filters=len(filter_clauses) > 0,)except Exception as e:- logger.warning("Failed to detect embedding models, using configured model", error=str(e))+ logger.warning(+ "Failed to detect embedding models, using configured model", error=str(e)+ )available_models = [embedding_model]# Parallelize embedding generation for all models@@ -219,9 +212,9 @@ class SearchService:model=formatted_model, input=[query])# Try to get embedding - some providers return .embedding, others return ['embedding']- embedding = getattr(resp.data[0], 'embedding', None)+ embedding = getattr(resp.data[0], "embedding", None)if embedding is None:- embedding = resp.data[0]['embedding']+ embedding = resp.data[0]["embedding"]return model_name, embeddingexcept Exception as e:last_exception = e@@ -232,9 +225,7 @@ class SearchService:attempts=attempts,error=str(e),)- raise RuntimeError(- f"Failed to embed with model {model_name}"- ) from e+ raise RuntimeError(f"Failed to embed with model {model_name}") from elogger.warning("Retrying embedding generation",@@ -247,9 +238,7 @@ class SearchService:delay = min(delay * 2, EMBED_RETRY_MAX_DELAY)# Should not reach here, but guard in case- raise RuntimeError(- f"Failed to embed with model {model_name}"- ) from last_exception+ raise RuntimeError(f"Failed to embed with model {model_name}") from last_exception# Run all embeddings in parallel, tolerating per-model failures so# one broken model (e.g. provider credentials removed after ingest)@@ -298,9 +287,7 @@ class SearchService:if len(values) == 0:# Empty array means "match nothing" - use impossible filter- filter_clauses.append(- {"term": {field_name: "__IMPOSSIBLE_VALUE__"}}- )+ filter_clauses.append({"term": {field_name: "__IMPOSSIBLE_VALUE__"}})elif len(values) == 1:# Single value filterfilter_clauses.append({"term": {field_name: values[0]}})@@ -324,15 +311,17 @@ class SearchService:for model_name, embedding_vector in query_embeddings.items():field_name = get_embedding_field_name(model_name)embedding_fields_to_check.append(field_name)- knn_queries.append({- "knn": {- field_name: {- "vector": embedding_vector,- "k": 50,- "num_candidates": 1000,+ knn_queries.append(+ {+ "knn": {+ field_name: {+ "vector": embedding_vector,+ "k": 50,+ "num_candidates": 1000,+ }}}- })+ )# Only require an embedding field when we actually have embeddings# to match against — otherwise we'd filter out every doc in keyword@@ -345,12 +334,14 @@ class SearchService:# Allow them through by matching on their embedding_model value.if failed_models:exists_should.append({"terms": {"embedding_model": failed_models}})- all_filters.append({- "bool": {- "should": exists_should,- "minimum_should_match": 1,+ all_filters.append(+ {+ "bool": {+ "should": exists_should,+ "minimum_should_match": 1,+ }}- })+ )logger.debug("Building hybrid query with filters",@@ -365,39 +356,43 @@ class SearchService:# the multi_match boost so keyword scoring isn't artificially damped.should_clauses = []if knn_queries:- should_clauses.append({- "dis_max": {- "tie_breaker": 0.0, # Take only the best match, no blending- "boost": 0.7, # 70% weight for semantic search- "queries": knn_queries,- }- })- should_clauses.extend([- {- "multi_match": {- "query": query,- "fields": ["text^2", "filename^1.5"],- "type": "best_fields",- "operator": "or",- "fuzziness": "AUTO:4,7",- "boost": 0.3 if knn_queries else 1.0,+ should_clauses.append(+ {+ "dis_max": {+ "tie_breaker": 0.0, # Take only the best match, no blending+ "boost": 0.7, # 70% weight for semantic search+ "queries": knn_queries,+ }}- },- {- # Prefix fallback for partial input (e.g. "vita" -> "vitamin").- # Avoid bool_prefix here because our current mappings are:- # - text: standard "text" (not search_as_you_type / edge-ngram)- # - filename: "keyword"- # match_phrase_prefix with a bounded expansion is safer.- "match_phrase_prefix": {- "text": {+ )+ should_clauses.extend(+ [+ {+ "multi_match": {"query": query,- "max_expansions": 50,- "boost": 0.25,+ "fields": ["text^2", "filename^1.5"],+ "type": "best_fields",+ "operator": "or",+ "fuzziness": "AUTO:4,7",+ "boost": 0.3 if knn_queries else 1.0,}- }- },- ])+ },+ {+ # Prefix fallback for partial input (e.g. "vita" -> "vitamin").+ # Avoid bool_prefix here because our current mappings are:+ # - text: standard "text" (not search_as_you_type / edge-ngram)+ # - filename: "keyword"+ # match_phrase_prefix with a bounded expansion is safer.+ "match_phrase_prefix": {+ "text": {+ "query": query,+ "max_expansions": 50,+ "boost": 0.25,+ }+ }+ },+ ]+ )query_block = {"bool": {@@ -445,9 +440,9 @@ class SearchService:if not is_wildcard_match_all and query_embeddings:try:fallback_search_body = copy.deepcopy(search_body)- knn_query_blocks = (- fallback_search_body["query"]["bool"]["should"][0]["dis_max"]["queries"]- )+ knn_query_blocks = fallback_search_body["query"]["bool"]["should"][0]["dis_max"][+ "queries"+ ]for query_candidate in knn_query_blocks:knn_section = query_candidate.get("knn")if isinstance(knn_section, dict):@@ -457,7 +452,7 @@ class SearchService:except (KeyError, IndexError, AttributeError, TypeError):fallback_search_body = None- # Authentication required - DLS will handle document filtering automatically+ # Authentication required - ACL filter is applied at the application layer abovelogger.debug("search_service authentication info",user_id=user_id,@@ -468,12 +463,14 @@ class SearchService:return {"results": [], "error": "Authentication required"}# Get user's OpenSearch client with JWT for OIDC auth through session manager- opensearch_client = self.session_manager.get_user_opensearch_client(- user_id, jwt_token- )+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)from opensearchpy.exceptions import RequestError- from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error, DISK_SPACE_ERROR_MESSAGE+ from utils.opensearch_utils import (+ OpenSearchDiskSpaceError,+ is_disk_space_error,+ DISK_SPACE_ERROR_MESSAGE,+ )search_params = {"terminate_after": 0}@@ -531,9 +528,7 @@ class SearchService:error=str(e),)raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e- logger.error(- "OpenSearch query failed", error=str(e), search_body=search_body- )+ logger.error("OpenSearch query failed", error=str(e), search_body=search_body)# Re-raise the exception so the API returns the error to frontendraise@@ -566,11 +561,7 @@ class SearchService:# to avoid broad semantic spillover for unique lookups.normalized_query = query.strip().lower()aggregations = results.get("aggregations", {})- if (- normalized_query- and not is_wildcard_match_all- and len(normalized_query) >= 4- ):+ if normalized_query and not is_wildcard_match_all and len(normalized_query) >= 4:exact_files = {filenamefor chunk in chunks@@ -598,8 +589,7 @@ class SearchService:"doc_count_error_upper_bound": 0,"sum_other_doc_count": 0,"buckets": [- {"key": key, "doc_count": count}- for key, count in counts.most_common()+ {"key": key, "doc_count": count} for key, count in counts.most_common()],}@@ -673,4 +663,4 @@ class SearchService:set_search_limit(limit)set_score_threshold(score_threshold)- return await self.search_tool(query, embedding_model=embedding_model)\ No newline at end of file+ return await self.search_tool(query, embedding_model=embedding_model)diff --git a/tests/unit/api/test_acl.py b/tests/unit/api/test_acl.pynew file mode 100644index 000000000..72383c0a2--- /dev/null+++ b/tests/unit/api/test_acl.py@@ -0,0 +1,68 @@+import json+import sys+from pathlib import Path+from unittest.mock import AsyncMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++class FakeSessionManager:+ def __init__(self, opensearch_client):+ self.opensearch_client = opensearch_client++ def get_user_opensearch_client(self, user_id, jwt_token):+ return self.opensearch_client+++class FakeUser:+ user_id = "reader"+ jwt_token = "token"+++async def test_get_document_acl_returns_read_only_acl_for_document():+ from api import acl as acl_module++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={+ "hits": {+ "hits": [+ {+ "_source": {+ "owner": "owner",+ "allowed_users": ["reader"],+ "allowed_groups": ["engineering"],+ }+ }+ ]+ }+ }+ )++ response = await acl_module.get_document_acl(+ "roadmap.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user=FakeUser(),+ )++ assert response.status_code == 200+ assert json.loads(response.body) == {+ "owner": "owner",+ "allowed_users": ["reader"],+ "allowed_groups": ["engineering"],+ }+ search_body = opensearch_client.search.await_args.kwargs["body"]+ assert search_body["_source"] == ["owner", "allowed_users", "allowed_groups"]+++def test_document_share_methods_are_not_exposed():+ from api import acl as acl_module++ assert not hasattr(acl_module, "share_document")+ assert not hasattr(acl_module, "unshare_document")diff --git a/tests/unit/test_query_acl_filtering.py b/tests/unit/test_query_acl_filtering.pynew file mode 100644index 000000000..4917996ff--- /dev/null+++ b/tests/unit/test_query_acl_filtering.py@@ -0,0 +1,45 @@+from pathlib import Path++from src.services.file_service import FileService+++def test_file_service_query_omits_application_acl_filter():+ query = FileService()._build_filter_query(+ user_id="user-123",+ connector_type="google_drive",+ mimetype="application/pdf",+ owner="[email protected]",+ search="roadmap",+ )++ filters = query["bool"]["filter"]++ assert filters == [+ {"term": {"connector_type": "google_drive"}},+ {"term": {"mimetype": "application/pdf"}},+ {"term": {"owner": "[email protected]"}},+ ]+ assert query["bool"]["must"] == [+ {+ "bool": {+ "should": [+ {"wildcard": {"filename": {"value": "*roadmap*"}}},+ {"prefix": {"filename": "roadmap"}},+ ],+ "minimum_should_match": 1,+ }+ }+ ]+++def test_service_query_paths_do_not_apply_document_visibility_filters():+ repo_root = Path(__file__).resolve().parents[2]+ helper_name = "build" + "_acl_filter"+ query_path_files = [+ repo_root / "src/services/file_service.py",+ repo_root / "src/services/search_service.py",+ ]++ for source_file in query_path_files:+ source = source_file.read_text()+ assert helper_name not in source